From dd3ea8d31b1b5615bcdd6b671f5e6e86315e8dda Mon Sep 17 00:00:00 2001 From: liyang830 Date: Fri, 10 Mar 2023 20:52:27 +0800 Subject: [PATCH 0001/1165] feat: modify materalized view query, check inner table structure --- src/Storages/StorageMaterializedView.cpp | 13 +++++++++++++ ...erialized_view_query_has_inner_table.reference | 2 ++ ...er_materialized_view_query_has_inner_table.sql | 15 +++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.reference create mode 100644 tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.sql diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index ae3fa62b38c..0bbd689043c 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -269,6 +270,18 @@ void StorageMaterializedView::alter( DatabaseCatalog::instance().updateViewDependency(old_select.select_table_id, table_id, new_select.select_table_id, table_id); new_metadata.setSelectQuery(new_select); + + /// check materialized view inner table structure + if (has_inner_table) + { + const Block & block = InterpreterSelectWithUnionQuery::getSampleBlock(new_select.select_query, local_context); + for (const auto & col : block.getColumnsWithTypeAndName()) + { + if (!tryGetTargetTable()->getInMemoryMetadata().columns.has(col.name)) + throw Exception(ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW, "column {} is not in materialized view inner table", col.name); + } + } + } /// end modify query diff --git a/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.reference b/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.reference new file mode 100644 index 00000000000..1191247b6d9 --- /dev/null +++ b/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.reference @@ -0,0 +1,2 @@ +1 +2 diff --git a/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.sql b/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.sql new file mode 100644 index 00000000000..73bbac59a95 --- /dev/null +++ b/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS src_table; +DROP TABLE IF EXISTS mv; + +CREATE TABLE src_table (`a` UInt32, `b` UInt32) ENGINE = MergeTree ORDER BY a; +CREATE MATERIALIZED VIEW mv UUID '2bad6d75-86fe-4da0-815b-2c7410253941' (`a` UInt32) ENGINE = MergeTree ORDER BY a AS SELECT a FROM src_table; + +INSERT INTO src_table (a, b) VALUES (1, 1), (2, 2); + +SELECT * FROM mv; + +SET allow_experimental_alter_materialized_view_structure = 1; +ALTER TABLE mv MODIFY QUERY SELECT a, b FROM src_table; -- {serverError QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW} + +DROP TABLE src_table; +DROP TABLE mv; \ No newline at end of file From 3f5853b970dd205465a5593d5786c1f8a4d82cc7 Mon Sep 17 00:00:00 2001 From: AN Date: Fri, 27 Oct 2023 19:17:13 +0300 Subject: [PATCH 0002/1165] Update index.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Punctuation fixes, узел→сервер as suggested by alexei-milovidov at https://github.com/ClickHouse/ClickHouse/pull/56040#issuecomment-1783155867, консистентность → согласованность (standard translation instead of calque) --- docs/ru/index.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/ru/index.md b/docs/ru/index.md index 78bb382753b..a9a666b18db 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -41,7 +41,7 @@ ClickHouse — столбцовая система управления база Разный порядок хранения данных лучше подходит для разных сценариев работы. Сценарий работы с данными — это то, какие производятся запросы, как часто и в каком соотношении; сколько читается данных на запросы каждого вида — строк, столбцов, байтов; как соотносятся чтения и обновления данных; какой рабочий размер данных и насколько локально он используется; используются ли транзакции и с какой изолированностью; какие требования к дублированию данных и логической целостности; требования к задержкам на выполнение и пропускной способности запросов каждого вида и т. п. -Чем больше нагрузка на систему, тем более важной становится специализация под сценарий работы, и тем более конкретной становится эта специализация. Не существует системы, одинаково хорошо подходящей под существенно различные сценарии работы. Если система подходит под широкое множество сценариев работы, то при достаточно большой нагрузке, система будет справляться со всеми сценариями работы плохо, или справляться хорошо только с одним из сценариев работы. +Чем больше нагрузка на систему, тем более важной становится специализация под сценарий работы, и тем более конкретной становится эта специализация. Не существует системы, одинаково хорошо подходящей под существенно различные сценарии работы. Если система подходит под широкое множество сценариев работы, то при достаточно большой нагрузке система будет справляться со всеми сценариями работы плохо, или справляться хорошо только с одним из сценариев работы. ## Ключевые особенности OLAP-сценария работы {#kliuchevye-osobennosti-olap-stsenariia-raboty} @@ -53,11 +53,11 @@ ClickHouse — столбцовая система управления база - запросы идут сравнительно редко (обычно не более сотни в секунду на сервер); - при выполнении простых запросов, допустимы задержки в районе 50 мс; - значения в столбцах достаточно мелкие — числа и небольшие строки (например, 60 байт на URL); -- требуется высокая пропускная способность при обработке одного запроса (до миллиардов строк в секунду на один узел); +- требуется высокая пропускная способность при обработке одного запроса (до миллиардов строк в секунду на один сервер); - транзакции отсутствуют; -- низкие требования к консистентности данных; -- в запросе одна большая таблица, все таблицы кроме одной маленькие; -- результат выполнения запроса существенно меньше исходных данных — то есть данные фильтруются или агрегируются; результат выполнения помещается в оперативную память одного узла. +- низкие требования к согласованности данных; +- в запросе одна большая таблица, все остальные таблицы из запроса — маленькие; +- результат выполнения запроса существенно меньше исходных данных — то есть данные фильтруются или агрегируются; результат выполнения помещается в оперативную память одного сервера. Легко видеть, что OLAP-сценарий работы существенно отличается от других распространённых сценариев работы (например, OLTP или Key-Value сценариев работы). Таким образом, не имеет никакого смысла пытаться использовать OLTP-системы или системы класса «ключ — значение» для обработки аналитических запросов, если вы хотите получить приличную производительность («выше плинтуса»). Например, если вы попытаетесь использовать для аналитики MongoDB или Redis — вы получите анекдотически низкую производительность по сравнению с OLAP-СУБД. @@ -77,11 +77,11 @@ ClickHouse — столбцовая система управления база ### По вводу-выводу {#po-vvodu-vyvodu} -1. Для выполнения аналитического запроса, требуется прочитать небольшое количество столбцов таблицы. В столбцовой БД для этого можно читать только нужные данные. Например, если вам требуется только 5 столбцов из 100, то следует рассчитывать на 20-кратное уменьшение ввода-вывода. -2. Так как данные читаются пачками, то их проще сжимать. Данные, лежащие по столбцам также лучше сжимаются. За счёт этого, дополнительно уменьшается объём ввода-вывода. -3. За счёт уменьшения ввода-вывода, больше данных влезает в системный кэш. +1. Для выполнения аналитического запроса требуется прочитать небольшое количество столбцов таблицы. В столбцовой БД для этого можно читать только нужные данные. Например, если вам требуется только 5 столбцов из 100, то следует рассчитывать на 20-кратное уменьшение ввода-вывода. +2. Так как данные читаются пачками, то их проще сжимать. Данные, лежащие по столбцам, также лучше сжимаются. За счёт этого, дополнительно уменьшается объём ввода-вывода. +3. За счёт уменьшения ввода-вывода больше данных влезает в системный кэш. -Например, для запроса «посчитать количество записей для каждой рекламной системы», требуется прочитать один столбец «идентификатор рекламной системы», который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия, возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть, такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике, такая скорость действительно достигается. +Например, для запроса «посчитать количество записей для каждой рекламной системы» требуется прочитать один столбец «идентификатор рекламной системы», который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике такая скорость действительно достигается. ### По вычислениям {#po-vychisleniiam} From 4dcbd6775a9cd1afe3c8be96e3c68c397ae547f0 Mon Sep 17 00:00:00 2001 From: Thom O'Connor Date: Fri, 3 Nov 2023 09:46:35 -0600 Subject: [PATCH 0003/1165] Update kill.md Added additional examples and context for killing queries and mutations --- docs/en/sql-reference/statements/kill.md | 64 ++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 294724dfa50..32de7a41e72 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -21,6 +21,35 @@ The queries to terminate are selected from the system.processes table using the Examples: +First, you'll need to get the list of incomplete queries. This SQL queries provides them according to those running the longest: + +List from a single ClickHouse node: +``` sql +SELECT + initial_query_id, + query_id, + formatReadableTimeDelta(elapsed) AS time_delta, + query, + * + FROM system.processes + WHERE query ILIKE 'SELECT%' + ORDER BY time_delta DESC; +``` + +List from a ClickHouse cluster: +``` sql +SELECT + initial_query_id, + query_id, + formatReadableTimeDelta(elapsed) AS time_delta, + query, + * + FROM clusterAllReplicas(default, system.processes) + WHERE query ILIKE 'SELECT%' + ORDER BY time_delta DESC; +``` + +Kill the query: ``` sql -- Forcibly terminates all queries with the specified query_id: KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90' @@ -44,6 +73,8 @@ A test query (`TEST`) only checks the user’s rights and displays a list of que ## KILL MUTATION +One of the first things to check if a ClickHouse system or service is not running well is for long-running, incomplete mutations. The asynchronous (background) nature of mutations can cause a large queue of them that can then consume all available resources on the service. You may need to either pause all new mutations, INSERTs, and SELECTs and allow the queue of mutations to complete, or else manually kill some of these mutations. + ``` sql KILL MUTATION [ON CLUSTER cluster] WHERE @@ -57,6 +88,39 @@ A test query (`TEST`) only checks the user’s rights and displays a list of mut Examples: +Get a count() of the number of incomplete mutations: + +Count of mutations from a single ClickHouse node: +``` sql +SELECT count(*) +FROM system.mutations +WHERE is_done = 0; +``` + +Count of mutations from a ClickHouse cluster of replicas: +``` sql +SELECT count(*) +FROM clusterAllReplicas('default',system.mutations) +WHERE is_done = 0; +``` + +Query the list of incomplete mutations: + +List of mutations from a single ClickHouse node: +``` sql +SELECT mutation_id,* +FROM system.mutations +WHERE is_done = 0; +``` + +List of mutations from a ClickHouse cluster: +``` sql +SELECT mutation_id,* +FROM clusterAllReplicas('default',system.mutations) +WHERE is_done = 0; +``` + +Kill the mutations as needed: ``` sql -- Cancel and remove all mutations of the single table: KILL MUTATION WHERE database = 'default' AND table = 'table' From 1134af19caeaffcf70cc94146faed346d6af0cf6 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Tue, 7 Nov 2023 22:33:29 -0800 Subject: [PATCH 0004/1165] [Docs] Fix typo Co-authored-by: Johnny <9611008+johnnymatthews@users.noreply.github.com> --- docs/en/sql-reference/statements/kill.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 32de7a41e72..a7d050e548c 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -21,7 +21,7 @@ The queries to terminate are selected from the system.processes table using the Examples: -First, you'll need to get the list of incomplete queries. This SQL queries provides them according to those running the longest: +First, you'll need to get the list of incomplete queries. This SQL query provides them according to those running the longest: List from a single ClickHouse node: ``` sql From 7e0d95e48cb399c047c9756d81b0f76ce67ea57f Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Tue, 7 Nov 2023 22:33:43 -0800 Subject: [PATCH 0005/1165] [Docs] Formatting Co-authored-by: Johnny <9611008+johnnymatthews@users.noreply.github.com> --- docs/en/sql-reference/statements/kill.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index a7d050e548c..57448c4f441 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -88,7 +88,7 @@ A test query (`TEST`) only checks the user’s rights and displays a list of mut Examples: -Get a count() of the number of incomplete mutations: +Get a `count()` of the number of incomplete mutations: Count of mutations from a single ClickHouse node: ``` sql From ec02a2a2c4f4d4a279732df2c2dd61ab8b0cb80a Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Tue, 7 Nov 2023 22:36:49 -0800 Subject: [PATCH 0006/1165] [Docs] Reword for clarity Co-authored-by: Johnny <9611008+johnnymatthews@users.noreply.github.com> --- docs/en/sql-reference/statements/kill.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 57448c4f441..c85870fc0c8 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -73,7 +73,10 @@ A test query (`TEST`) only checks the user’s rights and displays a list of que ## KILL MUTATION -One of the first things to check if a ClickHouse system or service is not running well is for long-running, incomplete mutations. The asynchronous (background) nature of mutations can cause a large queue of them that can then consume all available resources on the service. You may need to either pause all new mutations, INSERTs, and SELECTs and allow the queue of mutations to complete, or else manually kill some of these mutations. +The presence of long-running or incomplete mutations often indicates that a ClickHouse service is running poorly. The asynchronous nature of mutations can cause them to consume all available resources on a system. You may need to either: + +- Pause all new mutations, `INSERT`s , and `SELECT`s and allow the queue of mutations to complete. +- Or manually kill some of these mutations by sending a `KILLSIG` command. ``` sql KILL MUTATION [ON CLUSTER cluster] From 039bb1d599a5262e558b9b4ebd66fd85469afa3c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 22 Jan 2024 20:26:28 +0100 Subject: [PATCH 0007/1165] fix race on Context::async_insert_queue --- src/Interpreters/Context.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 7e89c794712..51cfd302338 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -181,6 +181,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; extern const int CLUSTER_DOESNT_EXIST; + extern const int ABORTED; } #define SHUTDOWN(log, desc, ptr, method) do \ @@ -556,7 +557,12 @@ struct ContextSharedPart : boost::noncopyable return; /// Need to flush the async insert queue before shutting down the database catalog - async_insert_queue.reset(); + std::shared_ptr delete_async_insert_queue; + { + std::lock_guard lock(mutex); + delete_async_insert_queue = std::move(async_insert_queue); + } + delete_async_insert_queue.reset(); /// Stop periodic reloading of the configuration files. /// This must be done first because otherwise the reloading may pass a changed config @@ -4838,11 +4844,15 @@ PartUUIDsPtr Context::getIgnoredPartUUIDs() const AsynchronousInsertQueue * Context::getAsynchronousInsertQueue() const { - return shared->async_insert_queue.get(); + std::lock_guard lock(mutex); + if (auto res = shared->async_insert_queue.get()) + return res; + throw Exception(ErrorCodes::ABORTED, "AsynchronousInsertQueue is not initialized yet or has been already shutdown"); } void Context::setAsynchronousInsertQueue(const std::shared_ptr & ptr) { + std::lock_guard lock(mutex); using namespace std::chrono; if (std::chrono::milliseconds(settings.async_insert_busy_timeout_ms) == 0ms) From eb881667638524f182f06e19d699704ce9e86196 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Jan 2024 00:28:28 +0100 Subject: [PATCH 0008/1165] Update Context.cpp --- src/Interpreters/Context.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 51cfd302338..217b247c21c 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -181,7 +181,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; extern const int CLUSTER_DOESNT_EXIST; - extern const int ABORTED; } #define SHUTDOWN(log, desc, ptr, method) do \ @@ -4845,9 +4844,7 @@ PartUUIDsPtr Context::getIgnoredPartUUIDs() const AsynchronousInsertQueue * Context::getAsynchronousInsertQueue() const { std::lock_guard lock(mutex); - if (auto res = shared->async_insert_queue.get()) - return res; - throw Exception(ErrorCodes::ABORTED, "AsynchronousInsertQueue is not initialized yet or has been already shutdown"); + return shared->async_insert_queue.get(); } void Context::setAsynchronousInsertQueue(const std::shared_ptr & ptr) From 4cfc8d1a34342d44adbc7d9c8c3a4916670d68b2 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Jan 2024 00:30:42 +0100 Subject: [PATCH 0009/1165] better method name --- src/Interpreters/Context.cpp | 2 +- src/Interpreters/Context.h | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 2 +- src/Interpreters/executeQuery.cpp | 2 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/System/StorageSystemAsynchronousInserts.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 217b247c21c..ab42e6b0ec9 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4841,7 +4841,7 @@ PartUUIDsPtr Context::getIgnoredPartUUIDs() const return ignored_part_uuids; } -AsynchronousInsertQueue * Context::getAsynchronousInsertQueue() const +AsynchronousInsertQueue * Context::tryGetAsynchronousInsertQueue() const { std::lock_guard lock(mutex); return shared->async_insert_queue.get(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 65566876a80..6180bfbde88 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1203,7 +1203,7 @@ public: PartUUIDsPtr getPartUUIDs() const; PartUUIDsPtr getIgnoredPartUUIDs() const; - AsynchronousInsertQueue * getAsynchronousInsertQueue() const; + AsynchronousInsertQueue * tryGetAsynchronousInsertQueue() const; void setAsynchronousInsertQueue(const std::shared_ptr & ptr); ReadTaskCallback getReadTaskCallback() const; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 1712c9608bf..f478b43049f 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -692,7 +692,7 @@ BlockIO InterpreterSystemQuery::execute() case Type::FLUSH_ASYNC_INSERT_QUEUE: { getContext()->checkAccess(AccessType::SYSTEM_FLUSH_ASYNC_INSERT_QUEUE); - auto * queue = getContext()->getAsynchronousInsertQueue(); + auto * queue = getContext()->tryGetAsynchronousInsertQueue(); if (!queue) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot flush asynchronous insert queue because it is not initialized"); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 4b5a6a84e17..a84c957d9a8 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -924,7 +924,7 @@ static std::tuple executeQueryImpl( std::unique_ptr interpreter; bool async_insert = false; - auto * queue = context->getAsynchronousInsertQueue(); + auto * queue = context->tryGetAsynchronousInsertQueue(); auto * logger = &Poco::Logger::get("executeQuery"); if (insert_query && async_insert_enabled) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index fa7206eeaac..9bc6c3872fd 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -907,7 +907,7 @@ void TCPHandler::processInsertQuery() Block processed_block; const auto & settings = query_context->getSettingsRef(); - auto * insert_queue = query_context->getAsynchronousInsertQueue(); + auto * insert_queue = query_context->tryGetAsynchronousInsertQueue(); const auto & insert_query = assert_cast(*state.parsed_query); bool async_insert_enabled = settings.async_insert; diff --git a/src/Storages/System/StorageSystemAsynchronousInserts.cpp b/src/Storages/System/StorageSystemAsynchronousInserts.cpp index 20ba4d1cdfb..b480821f8ea 100644 --- a/src/Storages/System/StorageSystemAsynchronousInserts.cpp +++ b/src/Storages/System/StorageSystemAsynchronousInserts.cpp @@ -34,7 +34,7 @@ void StorageSystemAsynchronousInserts::fillData(MutableColumns & res_columns, Co { using namespace std::chrono; - auto * insert_queue = context->getAsynchronousInsertQueue(); + auto * insert_queue = context->tryGetAsynchronousInsertQueue(); if (!insert_queue) return; From d970cd5b616ed8facab1c95e524940fe1a8502bf Mon Sep 17 00:00:00 2001 From: zhongyuankai <872237106@qq.com> Date: Tue, 9 Jan 2024 18:11:03 +0800 Subject: [PATCH 0010/1165] drop tables --- docs/en/sql-reference/statements/drop.md | 4 +- src/Interpreters/InterpreterDropQuery.cpp | 92 ++++++++++++++++--- src/Interpreters/InterpreterDropQuery.h | 4 + src/Parsers/ASTDropQuery.cpp | 20 +++- src/Parsers/ASTDropQuery.h | 3 + src/Parsers/ParserCreateQuery.cpp | 2 +- src/Parsers/ParserDropQuery.cpp | 21 ++--- src/Parsers/tests/gtest_dictionary_parser.cpp | 20 +++- .../0_stateless/02961_drop_tables.reference | 8 ++ .../queries/0_stateless/02961_drop_tables.sql | 32 +++++++ 10 files changed, 171 insertions(+), 35 deletions(-) create mode 100644 tests/queries/0_stateless/02961_drop_tables.reference create mode 100644 tests/queries/0_stateless/02961_drop_tables.sql diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 8ed00f625d6..159ab09ab94 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -30,9 +30,11 @@ Also see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md) Syntax: ``` sql -DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY] [db.]name [ON CLUSTER cluster] [SYNC] +DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY] [db1.]name_1[, [db2.]name_2, ...] [ON CLUSTER cluster] [SYNC] ``` +Note that deleting multiple tables at the same time is a non-atomic deletion. If a table fails to be deleted, subsequent tables will not be deleted. + ## DROP DICTIONARY Deletes the dictionary. diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 711100b5de1..3285e29d22a 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -53,16 +54,56 @@ InterpreterDropQuery::InterpreterDropQuery(const ASTPtr & query_ptr_, ContextMut { } - BlockIO InterpreterDropQuery::execute() { auto & drop = query_ptr->as(); + if (drop.database_and_tables) + { + BlockIO res; + auto & database_and_tables = drop.database_and_tables->as(); + for (const auto & child : database_and_tables.children) + { + auto cloned = drop.clone(); + auto & query = cloned->as(); + query.database_and_tables = nullptr; - if (!drop.cluster.empty() && drop.table && !drop.if_empty && !maybeRemoveOnCluster(query_ptr, getContext())) + auto database_and_table = dynamic_pointer_cast(child); + if (database_and_table->name_parts.size() == 2) + { + query.database = std::make_shared(database_and_table->name_parts[0]); + query.table = std::make_shared(database_and_table->name_parts[1]); + } + else + { + query.table = std::make_shared(database_and_table->name_parts[0]); + } + + if (query.database) + query.children.push_back(query.database); + + if (query.table) + query.children.push_back(query.table); + + current_query_ptr = cloned; + res = executeSingleDropQuery(cloned); + } + return res; + } + else + { + current_query_ptr = query_ptr; + return executeSingleDropQuery(query_ptr); + } +} + +BlockIO InterpreterDropQuery::executeSingleDropQuery(const ASTPtr & drop_query_ptr) +{ + auto & drop = drop_query_ptr->as(); + if (!drop.cluster.empty() && drop.table && !drop.if_empty && !maybeRemoveOnCluster(current_query_ptr, getContext())) { DDLQueryOnClusterParams params; params.access_to_check = getRequiredAccessForDDLOnCluster(); - return executeDDLQueryOnCluster(query_ptr, getContext(), params); + return executeDDLQueryOnCluster(current_query_ptr, getContext(), params); } if (getContext()->getSettingsRef().database_atomic_wait_for_drop_and_detach_synchronously) @@ -70,11 +111,11 @@ BlockIO InterpreterDropQuery::execute() if (drop.table) return executeToTable(drop); - else if (drop.database && !drop.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext())) + else if (drop.database && !drop.cluster.empty() && !maybeRemoveOnCluster(current_query_ptr, getContext())) { - DDLQueryOnClusterParams params; - params.access_to_check = getRequiredAccessForDDLOnCluster(); - return executeDDLQueryOnCluster(query_ptr, getContext(), params); + DDLQueryOnClusterParams params; + params.access_to_check = getRequiredAccessForDDLOnCluster(); + return executeDDLQueryOnCluster(current_query_ptr, getContext(), params); } else if (drop.database) return executeToDatabase(drop); @@ -82,7 +123,6 @@ BlockIO InterpreterDropQuery::execute() throw Exception(ErrorCodes::LOGICAL_ERROR, "Nothing to drop, both names are empty"); } - void InterpreterDropQuery::waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait) { if (uuid_to_wait == UUIDHelpers::Nil) @@ -155,7 +195,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue table_id.uuid = database->tryGetTableUUID(table_id.table_name); /// Prevents recursive drop from drop database query. The original query must specify a table. - bool is_drop_or_detach_database = !query_ptr->as()->table; + bool is_drop_or_detach_database = !current_query_ptr->as()->table; AccessFlags drop_storage; @@ -178,7 +218,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue return executeDDLQueryOnCluster(new_query_ptr, getContext(), params); } - if (database->shouldReplicateQuery(getContext(), query_ptr)) + if (database->shouldReplicateQuery(getContext(), current_query_ptr)) { if (query.kind == ASTDropQuery::Kind::Detach) context_->checkAccess(drop_storage, table_id); @@ -248,7 +288,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue auto metadata_snapshot = table->getInMemoryMetadataPtr(); /// Drop table data, don't touch metadata - table->truncate(query_ptr, metadata_snapshot, context_, table_excl_lock); + table->truncate(current_query_ptr, metadata_snapshot, context_, table_excl_lock); } else if (query.kind == ASTDropQuery::Kind::Drop) { @@ -307,7 +347,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); /// Drop table data, don't touch metadata auto metadata_snapshot = table->getInMemoryMetadataPtr(); - table->truncate(query_ptr, metadata_snapshot, getContext(), table_lock); + table->truncate(current_query_ptr, metadata_snapshot, getContext(), table_lock); } else if (kind == ASTDropQuery::Kind::Drop) { @@ -440,11 +480,35 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, return {}; } +void InterpreterDropQuery::extendQueryLogElemImpl(DB::QueryLogElement & elem, const DB::ASTPtr & ast, DB::ContextPtr context_) const +{ + auto & drop = ast->as(); + if (drop.database_and_tables) + { + auto & list = drop.database_and_tables->as(); + for (auto it = list.children.begin(); it != list.children.end(); ++it) + { + auto identifier = dynamic_pointer_cast(*it); + if (identifier->name_parts.size() == 2) + { + auto quoted_database = backQuoteIfNeed(identifier->name_parts[0]); + elem.query_databases.insert(quoted_database); + elem.query_tables.insert(quoted_database + "." + backQuoteIfNeed(identifier->name_parts[1])); + } + else + { + auto quoted_database = backQuoteIfNeed(context_->getCurrentDatabase()); + elem.query_databases.insert(quoted_database); + elem.query_tables.insert(quoted_database + "." + backQuoteIfNeed(identifier->name_parts[0])); + } + } + } +} AccessRightsElements InterpreterDropQuery::getRequiredAccessForDDLOnCluster() const { AccessRightsElements required_access; - const auto & drop = query_ptr->as(); + const auto & drop = current_query_ptr->as(); if (!drop.table) { @@ -512,7 +576,7 @@ bool InterpreterDropQuery::supportsTransactions() const { /// Enable only for truncate table with MergeTreeData engine - auto & drop = query_ptr->as(); + auto & drop = current_query_ptr->as(); return drop.cluster.empty() && !drop.temporary diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 7ae544a7356..08668f47225 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -29,10 +29,14 @@ public: bool supportsTransactions() const override; + void extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & ast, ContextPtr context_) const override; + private: AccessRightsElements getRequiredAccessForDDLOnCluster() const; ASTPtr query_ptr; + ASTPtr current_query_ptr; + BlockIO executeSingleDropQuery(const ASTPtr & drop_query_ptr); BlockIO executeToDatabase(const ASTDropQuery & query); BlockIO executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait); diff --git a/src/Parsers/ASTDropQuery.cpp b/src/Parsers/ASTDropQuery.cpp index ad1294c6e71..aea5a00bbcd 100644 --- a/src/Parsers/ASTDropQuery.cpp +++ b/src/Parsers/ASTDropQuery.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -48,7 +50,7 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState settings.ostr << "TEMPORARY "; - if (!table && database) + if (!table && !database_and_tables && database) settings.ostr << "DATABASE "; else if (is_dictionary) settings.ostr << "DICTIONARY "; @@ -65,8 +67,22 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState settings.ostr << (settings.hilite ? hilite_none : ""); - if (!table && database) + if (!table && !database_and_tables && database) settings.ostr << backQuoteIfNeed(getDatabase()); + else if (database_and_tables) + { + auto & list = database_and_tables->as(); + for (auto it = list.children.begin(); it != list.children.end(); ++it) + { + if (it != list.children.begin()) + settings.ostr << ", "; + + auto identifier = dynamic_pointer_cast(*it); + settings.ostr << (identifier->name_parts.size() == 2 + ? backQuoteIfNeed(identifier->name_parts[0]) + "." + backQuoteIfNeed(identifier->name_parts[1]) + : backQuoteIfNeed(identifier->name_parts[0])); + } + } else settings.ostr << (database ? backQuoteIfNeed(getDatabase()) + "." : "") << backQuoteIfNeed(getTable()); diff --git a/src/Parsers/ASTDropQuery.h b/src/Parsers/ASTDropQuery.h index a732b354260..7aca9a883ed 100644 --- a/src/Parsers/ASTDropQuery.h +++ b/src/Parsers/ASTDropQuery.h @@ -37,6 +37,9 @@ public: // We detach the object permanently, so it will not be reattached back during server restart. bool permanently{false}; + /// Example: Drop TABLE t1, t2, t3... + ASTPtr database_and_tables; + /** Get the text that identifies this element. */ String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 1f6f68c9d8e..db11960cb59 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -104,7 +104,7 @@ bool ParserColumnDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & bool ParserNameList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - return ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) + return ParserList(std::make_unique(true, true), std::make_unique(TokenType::Comma), false) .parse(pos, node, expected); } diff --git a/src/Parsers/ParserDropQuery.cpp b/src/Parsers/ParserDropQuery.cpp index 450c8a1afec..b6a6d6e28d4 100644 --- a/src/Parsers/ParserDropQuery.cpp +++ b/src/Parsers/ParserDropQuery.cpp @@ -2,7 +2,7 @@ #include #include - +#include namespace DB { @@ -18,15 +18,17 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons ParserKeyword s_view("VIEW"); ParserKeyword s_database("DATABASE"); ParserToken s_dot(TokenType::Dot); + ParserToken s_comma(TokenType::Comma); ParserKeyword s_if_exists("IF EXISTS"); ParserKeyword s_if_empty("IF EMPTY"); ParserIdentifier name_p(true); ParserKeyword s_permanently("PERMANENTLY"); ParserKeyword s_no_delay("NO DELAY"); ParserKeyword s_sync("SYNC"); + ParserNameList tables_p; ASTPtr database; - ASTPtr table; + ASTPtr database_and_tables; String cluster_str; bool if_exists = false; bool if_empty = false; @@ -68,15 +70,8 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons if (s_if_empty.ignore(pos, expected)) if_empty = true; - if (!name_p.parse(pos, table, expected)) + if (!tables_p.parse(pos, database_and_tables, expected)) return false; - - if (s_dot.ignore(pos, expected)) - { - database = table; - if (!name_p.parse(pos, table, expected)) - return false; - } } /// common for tables / dictionaries / databases @@ -105,13 +100,13 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons query->sync = sync; query->permanently = permanently; query->database = database; - query->table = table; + query->database_and_tables = database_and_tables; if (database) query->children.push_back(database); - if (table) - query->children.push_back(table); + if (database_and_tables) + query->children.push_back(database_and_tables); query->cluster = cluster_str; diff --git a/src/Parsers/tests/gtest_dictionary_parser.cpp b/src/Parsers/tests/gtest_dictionary_parser.cpp index c0a975f7a38..020e145157c 100644 --- a/src/Parsers/tests/gtest_dictionary_parser.cpp +++ b/src/Parsers/tests/gtest_dictionary_parser.cpp @@ -300,9 +300,20 @@ TEST(ParserDictionaryDDL, ParseDropQuery) ASTPtr ast1 = parseQuery(parser, input1.data(), input1.data() + input1.size(), "", 0, 0); ASTDropQuery * drop1 = ast1->as(); + auto get_database_and_table = [](const ASTDropQuery & drop) -> std::pair + { + auto & database_and_tables = drop.database_and_tables->as(); + auto database_and_table = dynamic_pointer_cast(database_and_tables.children[0]); + if (database_and_table->name_parts.size() == 2) + return {database_and_table->name_parts[0], database_and_table->name_parts[1]}; + else + return {"", database_and_table->name_parts[0]}; + }; + EXPECT_TRUE(drop1->is_dictionary); - EXPECT_EQ(drop1->getDatabase(), "test"); - EXPECT_EQ(drop1->getTable(), "dict1"); + auto [database1, table1] = get_database_and_table(*drop1); + EXPECT_EQ(database1, "test"); + EXPECT_EQ(table1, "dict1"); auto str1 = serializeAST(*drop1); EXPECT_EQ(input1, str1); @@ -312,8 +323,9 @@ TEST(ParserDictionaryDDL, ParseDropQuery) ASTDropQuery * drop2 = ast2->as(); EXPECT_TRUE(drop2->is_dictionary); - EXPECT_EQ(drop2->getDatabase(), ""); - EXPECT_EQ(drop2->getTable(), "dict2"); + auto [database2, table2] = get_database_and_table(*drop2); + EXPECT_EQ(database2, ""); + EXPECT_EQ(table2, "dict2"); auto str2 = serializeAST(*drop2); EXPECT_EQ(input2, str2); } diff --git a/tests/queries/0_stateless/02961_drop_tables.reference b/tests/queries/0_stateless/02961_drop_tables.reference new file mode 100644 index 00000000000..c0465dc592a --- /dev/null +++ b/tests/queries/0_stateless/02961_drop_tables.reference @@ -0,0 +1,8 @@ +-- check which tables exist in 02961_db1 +-- check which tables exist in 02961_db2 +02961_tb4 +02961_tb5 +Test when deletion of existing table fails +-- check which tables exist in 02961_db1 +-- check which tables exist in 02961_db2 +02961_tb5 diff --git a/tests/queries/0_stateless/02961_drop_tables.sql b/tests/queries/0_stateless/02961_drop_tables.sql new file mode 100644 index 00000000000..e9695da5ed8 --- /dev/null +++ b/tests/queries/0_stateless/02961_drop_tables.sql @@ -0,0 +1,32 @@ +-- Tags: no-parallel-replicas +DROP DATABASE IF EXISTS 02961_db1; +CREATE DATABASE IF NOT EXISTS 02961_db1; +DROP DATABASE IF EXISTS 02961_db2; +CREATE DATABASE IF NOT EXISTS 02961_db2; + + +CREATE TABLE IF NOT EXISTS 02961_db1.02961_tb1 (id UInt32) Engine=Memory(); +CREATE TABLE IF NOT EXISTS 02961_db1.02961_tb2 (id UInt32) Engine=Memory(); + +CREATE TABLE IF NOT EXISTS 02961_db2.02961_tb3 (id UInt32) Engine=Memory(); +CREATE TABLE IF NOT EXISTS 02961_db2.02961_tb4 (id UInt32) Engine=Memory(); +CREATE TABLE IF NOT EXISTS 02961_db2.02961_tb5 (id UInt32) Engine=Memory(); + +DROP TABLE 02961_db1.02961_tb1, 02961_db1.02961_tb2, 02961_db2.02961_tb3; + +SELECT '-- check which tables exist in 02961_db1'; +SHOW TABLES FROM 02961_db1; +SELECT '-- check which tables exist in 02961_db2'; +SHOW TABLES FROM 02961_db2; + +SELECT 'Test when deletion of existing table fails'; +DROP TABLE 02961_db2.02961_tb4, 02961_db1.02961_tb1, 02961_db2.02961_tb5; -- { serverError UNKNOWN_TABLE } + +SELECT '-- check which tables exist in 02961_db1'; +SHOW TABLES FROM 02961_db1; +SELECT '-- check which tables exist in 02961_db2'; +SHOW TABLES FROM 02961_db2; + + +DROP DATABASE IF EXISTS 02961_db1; +DROP DATABASE IF EXISTS 02961_db2; From 915226d22b772191a82f7ca45826eab6f82dcccf Mon Sep 17 00:00:00 2001 From: zhongyuankai <872237106@qq.com> Date: Wed, 24 Jan 2024 13:26:38 +0800 Subject: [PATCH 0011/1165] fix test --- src/Interpreters/InterpreterDropQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 3285e29d22a..4a13ff4de77 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -576,12 +576,12 @@ bool InterpreterDropQuery::supportsTransactions() const { /// Enable only for truncate table with MergeTreeData engine - auto & drop = current_query_ptr->as(); + auto & drop = query_ptr->as(); return drop.cluster.empty() && !drop.temporary && drop.kind == ASTDropQuery::Kind::Truncate - && drop.table; + && drop.database_and_tables; } void registerInterpreterDropQuery(InterpreterFactory & factory) From 3b2a1dc21999496b3ad88c88e3a1dc52cb1a7ef7 Mon Sep 17 00:00:00 2001 From: zhongyuankai <54787696+zhongyuankai@users.noreply.github.com> Date: Sun, 28 Jan 2024 09:41:56 +0800 Subject: [PATCH 0012/1165] fix test --- src/Interpreters/InterpreterDropQuery.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 4a13ff4de77..6e0fda610ad 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -66,6 +66,7 @@ BlockIO InterpreterDropQuery::execute() auto cloned = drop.clone(); auto & query = cloned->as(); query.database_and_tables = nullptr; + query.children.clear(); auto database_and_table = dynamic_pointer_cast(child); if (database_and_table->name_parts.size() == 2) From 9a289a3baac9b2ed51462a9ff15a1c945acbca1f Mon Sep 17 00:00:00 2001 From: zhongyuankai <54787696+zhongyuankai@users.noreply.github.com> Date: Sun, 28 Jan 2024 13:53:38 +0800 Subject: [PATCH 0013/1165] fix test --- tests/queries/0_stateless/02961_drop_tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02961_drop_tables.sql b/tests/queries/0_stateless/02961_drop_tables.sql index e9695da5ed8..e91ac4bfe19 100644 --- a/tests/queries/0_stateless/02961_drop_tables.sql +++ b/tests/queries/0_stateless/02961_drop_tables.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel-replicas +-- Tags: no-parallel DROP DATABASE IF EXISTS 02961_db1; CREATE DATABASE IF NOT EXISTS 02961_db1; DROP DATABASE IF EXISTS 02961_db2; From 3de9dafa1493f90ce95d0a94cf65dc03a21516ab Mon Sep 17 00:00:00 2001 From: zhongyuankai <54787696+zhongyuankai@users.noreply.github.com> Date: Sun, 28 Jan 2024 16:12:51 +0800 Subject: [PATCH 0014/1165] fix build check --- src/Parsers/ASTDropQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTDropQuery.cpp b/src/Parsers/ASTDropQuery.cpp index aea5a00bbcd..f7b7c768abd 100644 --- a/src/Parsers/ASTDropQuery.cpp +++ b/src/Parsers/ASTDropQuery.cpp @@ -72,7 +72,7 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState else if (database_and_tables) { auto & list = database_and_tables->as(); - for (auto it = list.children.begin(); it != list.children.end(); ++it) + for (auto * it = list.children.begin(); it != list.children.end(); ++it) { if (it != list.children.begin()) settings.ostr << ", "; From f91feb0dcb405df80f317f456372c7374f2c75ee Mon Sep 17 00:00:00 2001 From: Daniil Ivanik Date: Tue, 30 Jan 2024 14:17:11 +0100 Subject: [PATCH 0015/1165] Initial working commit --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 210 ++++++++++-------- src/Storages/SelectQueryInfo.h | 2 +- src/Storages/System/StorageSystemNumbers.cpp | 8 +- src/Storages/System/StorageSystemNumbers.h | 36 +-- src/Storages/System/attachSystemTables.cpp | 5 +- src/TableFunctions/CMakeLists.txt | 2 +- src/TableFunctions/ITableFunction.cpp | 2 +- src/TableFunctions/TableFunctionNumbers.cpp | 2 +- .../TableFunctionsGenerateSeries.cpp | 100 +++++++++ src/TableFunctions/registerTableFunctions.cpp | 1 + src/TableFunctions/registerTableFunctions.h | 1 + 11 files changed, 234 insertions(+), 135 deletions(-) create mode 100644 src/TableFunctions/TableFunctionsGenerateSeries.cpp diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 5173b18c6bf..13a14ffb917 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -26,41 +26,59 @@ namespace class NumbersSource : public ISource { public: - NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_) - : ISource(createHeader()), block_size(block_size_), next(offset_), step(step_) + NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_, const std::string& column_name, UInt64 inner_step_) + : ISource(createHeader(column_name)), block_size(block_size_), next(offset_), step(step_), inner_step(inner_step_), inner_remainder(offset_ % inner_step_) { } String getName() const override { return "Numbers"; } - static Block createHeader() { return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "number")}; } + static Block createHeader(const std::string& column_name) { return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), column_name)}; } protected: Chunk generate() override { - auto column = ColumnUInt64::create(block_size); - ColumnUInt64::Container & vec = column->getData(); UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. + UInt64 first_element = (curr / inner_step) * inner_step + inner_remainder; + if (first_element < curr) { + first_element += inner_step; + } + UInt64 filtered_block_size = 0; + if (first_element - curr >= block_size) { + auto column = ColumnUInt64::create(0); + return {Columns{std::move(column)}, filtered_block_size}; + } + if (first_element - curr < block_size) { + filtered_block_size = (block_size - (first_element - curr) - 1) / inner_step + 1; + } + + auto column = ColumnUInt64::create(filtered_block_size); + ColumnUInt64::Container & vec = column->getData(); UInt64 * pos = vec.data(); /// This also accelerates the code. - UInt64 * end = &vec[block_size]; - iota(pos, static_cast(end - pos), curr); + UInt64 * end = &vec[filtered_block_size]; + iota(pos, static_cast(end - pos), UInt64{0}); + for (UInt64 p = 0; p < filtered_block_size; p += 1) { + vec[p] = vec[p] * inner_step + first_element; + } next += step; progress(column->size(), column->byteSize()); - return {Columns{std::move(column)}, block_size}; + return {Columns{std::move(column)}, filtered_block_size}; } private: UInt64 block_size; UInt64 next; UInt64 step; + UInt64 inner_step; + UInt64 inner_remainder; }; -UInt128 sizeOfRange(const Range & r) +[[maybe_unused]] UInt128 sizeOfRange(const Range & r) { UInt128 size; if (r.right.isPositiveInfinity()) @@ -77,7 +95,7 @@ UInt128 sizeOfRange(const Range & r) return size; }; -auto sizeOfRanges(const Ranges & rs) +[[maybe_unused]] auto sizeOfRanges(const Ranges & rs) { UInt128 total_size{}; for (const Range & r : rs) @@ -91,7 +109,7 @@ auto sizeOfRanges(const Ranges & rs) /// Generate numbers according to ranges. /// Numbers generated is ordered in one stream. /// Notice that we will not generate additional numbers out of ranges. -class NumbersRangedSource : public ISource +class [[maybe_unused]] NumbersRangedSource : public ISource { public: /// Represent a position in Ranges list. @@ -109,8 +127,8 @@ public: using RangesStatePtr = std::shared_ptr; - NumbersRangedSource(const Ranges & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_) - : ISource(NumbersSource::createHeader()), ranges(ranges_), ranges_state(ranges_state_), base_block_size(base_block_size_) + [[maybe_unused]] NumbersRangedSource(const Ranges & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, const std::string& column_name) + : ISource(NumbersSource::createHeader(column_name)), ranges(ranges_), ranges_state(ranges_state_), base_block_size(base_block_size_) { } @@ -273,7 +291,7 @@ private: namespace { /// Whether we should push limit down to scan. -bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) +[[maybe_unused]] bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) { const auto & query = query_info.query->as(); /// Just ignore some minor cases, such as: @@ -286,7 +304,7 @@ bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) /// Shrink ranges to size. /// For example: ranges: [1, 5], [8, 100]; size: 7, we will get [1, 5], [8, 9] -void shrinkRanges(Ranges & ranges, size_t size) +[[maybe_unused]] void shrinkRanges(Ranges & ranges, size_t size) { size_t last_range_idx = 0; for (size_t i = 0; i < ranges.size(); i++) @@ -375,107 +393,107 @@ Pipe ReadFromSystemNumbersStep::makePipe() num_streams = 1; /// Build rpn of query filters - KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); + // KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); Pipe pipe; Ranges ranges; - if (condition.extractPlainRanges(ranges)) - { - /// Intersect ranges with table range - std::optional table_range; - std::optional overflowed_table_range; + // if (condition.extractPlainRanges(ranges)) + // { + // /// Intersect ranges with table range + // std::optional table_range; + // std::optional overflowed_table_range; - if (numbers_storage.limit.has_value()) - { - if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) - { - table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); - } - /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) - else - { - table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); - auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); - overflowed_table_range.emplace( - FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); - } - } - else - { - table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); - } + // if (numbers_storage.limit.has_value()) + // { + // if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) + // { + // table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); + // } + // /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) + // else + // { + // table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); + // auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); + // overflowed_table_range.emplace( + // FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); + // } + // } + // else + // { + // table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); + // } - Ranges intersected_ranges; - for (auto & r : ranges) - { - auto intersected_range = table_range->intersectWith(r); - if (intersected_range) - intersected_ranges.push_back(*intersected_range); - } - /// intersection with overflowed_table_range goes back. - if (overflowed_table_range.has_value()) - { - for (auto & r : ranges) - { - auto intersected_range = overflowed_table_range->intersectWith(r); - if (intersected_range) - intersected_ranges.push_back(*overflowed_table_range); - } - } + // Ranges intersected_ranges; + // for (auto & r : ranges) + // { + // auto intersected_range = table_range->intersectWith(r); + // if (intersected_range) + // intersected_ranges.push_back(*intersected_range); + // } + // /// intersection with overflowed_table_range goes back. + // if (overflowed_table_range.has_value()) + // { + // for (auto & r : ranges) + // { + // auto intersected_range = overflowed_table_range->intersectWith(r); + // if (intersected_range) + // intersected_ranges.push_back(*overflowed_table_range); + // } + // } - /// ranges is blank, return a source who has no data - if (intersected_ranges.empty()) - { - pipe.addSource(std::make_shared(NumbersSource::createHeader())); - return pipe; - } - const auto & limit_length = limit_length_and_offset.first; - const auto & limit_offset = limit_length_and_offset.second; + // /// ranges is blank, return a source who has no data + // if (intersected_ranges.empty()) + // { + // pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); + // return pipe; + // } + // const auto & limit_length = limit_length_and_offset.first; + // const auto & limit_offset = limit_length_and_offset.second; - /// If intersected ranges is limited or we can pushdown limit. - if (!intersected_ranges.rbegin()->right.isPositiveInfinity() || should_pushdown_limit) - { - UInt128 total_size = sizeOfRanges(intersected_ranges); - UInt128 query_limit = limit_length + limit_offset; + // /// If intersected ranges is limited or we can pushdown limit. + // if (!intersected_ranges.rbegin()->right.isPositiveInfinity() || should_pushdown_limit) + // { + // UInt128 total_size = sizeOfRanges(intersected_ranges); + // UInt128 query_limit = limit_length + limit_offset; - /// limit total_size by query_limit - if (should_pushdown_limit && query_limit < total_size) - { - total_size = query_limit; - /// We should shrink intersected_ranges for case: - /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 - shrinkRanges(intersected_ranges, total_size); - } + // /// limit total_size by query_limit + // if (should_pushdown_limit && query_limit < total_size) + // { + // total_size = query_limit; + // /// We should shrink intersected_ranges for case: + // /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 + // shrinkRanges(intersected_ranges, total_size); + // } - checkLimits(size_t(total_size)); + // checkLimits(size_t(total_size)); - if (total_size / max_block_size < num_streams) - num_streams = static_cast(total_size / max_block_size); + // if (total_size / max_block_size < num_streams) + // num_streams = static_cast(total_size / max_block_size); - if (num_streams == 0) - num_streams = 1; + // if (num_streams == 0) + // num_streams = 1; - /// Ranges state, all streams will share the state. - auto ranges_state = std::make_shared(); - for (size_t i = 0; i < num_streams; ++i) - { - auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size); + // /// Ranges state, all streams will share the state. + // auto ranges_state = std::make_shared(); + // for (size_t i = 0; i < num_streams; ++i) + // { + // auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size, numbers_storage.column_name); - if (i == 0) - source->addTotalRowsApprox(total_size); + // if (i == 0) + // source->addTotalRowsApprox(total_size); - pipe.addSource(std::move(source)); - } - return pipe; - } - } + // pipe.addSource(std::move(source)); + // } + // return pipe; + // } + // } /// Fall back to NumbersSource for (size_t i = 0; i < num_streams; ++i) { auto source - = std::make_shared(max_block_size, numbers_storage.offset + i * max_block_size, num_streams * max_block_size); + = std::make_shared(max_block_size, numbers_storage.offset + i * max_block_size, num_streams * max_block_size, numbers_storage.column_name, numbers_storage.step); if (numbers_storage.limit && i == 0) { diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 662a5c0ef5a..2b4afaa6345 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -229,4 +229,4 @@ struct SelectQueryInfo bool isFinal() const; }; -} + } diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index b100be7cdf4..cd7207917a9 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -11,15 +11,16 @@ #include #include #include +#include namespace DB { -StorageSystemNumbers::StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, std::optional limit_, UInt64 offset_) - : IStorage(table_id), multithreaded(multithreaded_), limit(limit_), offset(offset_) +StorageSystemNumbers::StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, const std::string& column_name_, std::optional limit_, UInt64 offset_, UInt64 step_) + : IStorage(table_id), multithreaded(multithreaded_), limit(limit_), offset(offset_), column_name(column_name_), step(step_) { StorageInMemoryMetadata storage_metadata; - storage_metadata.setColumns(ColumnsDescription({{"number", std::make_shared()}})); + storage_metadata.setColumns(ColumnsDescription({{column_name_, std::make_shared()}})); setInMemoryMetadata(storage_metadata); } @@ -33,6 +34,7 @@ void StorageSystemNumbers::read( size_t max_block_size, size_t num_streams) { + // LOG_DEBUG(&Poco::Logger::get("Reading from SystemNumbers"), "Limit : {}", limit.value()); query_plan.addStep(std::make_unique( column_names, shared_from_this(), storage_snapshot, query_info, std::move(context), max_block_size, num_streams)); } diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index fe6227db406..ffe87b8ad14 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -10,39 +10,11 @@ namespace DB class Context; - -/** Implements a table engine for the system table "numbers". - * The table contains the only column number UInt64. - * From this table, you can read all natural numbers, starting from 0 (to 2^64 - 1, and then again). - * - * You could also specify a limit (how many numbers to give). - * - * How to generate numbers? - * - * 1. First try a smart fashion: - * - * In this fashion we try to push filters and limit down to scanning. - * Firstly extract plain ranges(no overlapping and ordered) by filter expressions. - * - * For example: - * where (numbers > 1 and numbers < 3) or (numbers in (4, 6)) or (numbers > 7 and numbers < 9) - * - * We will get ranges - * (1, 3), [4, 4], [6, 6], (7, 9) - * - * Then split the ranges evenly to one or multi-streams. With this way we will get result without large scanning. - * - * 2. If fail to extract plain ranges, fall back to ordinary scanning. - * - * If multithreaded is specified, numbers will be generated in several streams - * (and result could be out of order). If both multithreaded and limit are specified, - * the table could give you not exactly 1..limit range, but some arbitrary 'limit' numbers. - */ -class StorageSystemNumbers final : public IStorage +class StorageSystemNumbers final : public IStorage { public: /// Otherwise, streams concurrently increment atomic. - StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, std::optional limit_ = std::nullopt, UInt64 offset_ = 0); + StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, const std::string& column_name, std::optional limit_ = std::nullopt, UInt64 offset_ = 0, UInt64 step_ = 1); std::string getName() const override { return "SystemNumbers"; } @@ -67,6 +39,10 @@ private: bool multithreaded; std::optional limit; UInt64 offset; + std::string column_name; + + UInt64 step; + }; } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index bf898f57833..ddd89709b6a 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -118,8 +118,9 @@ namespace DB void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper) { attach(context, system_database, "one", "This table contains a single row with a single dummy UInt8 column containing the value 0. Used when the table is not specified explicitly, for example in queries like `SELECT 1`."); - attach(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false); - attach(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true); + attach(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "number"); + attach(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "number"); + // attach(context, system_database, "generate_series", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", false, "generate_series"); attach(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attach(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); attach(context, system_database, "databases", "Lists all databases of the current server."); diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index 770990cc405..c5c2a660935 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -29,7 +29,7 @@ if (TARGET ch_contrib::azure_sdk) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::azure_sdk) endif () -if (TARGET ch_contrib::simdjson) +if (TARGET ch_co`trib::simdjson) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::simdjson) endif () diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index 137e1dc27fe..c854b6b0f9c 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -5,7 +5,7 @@ #include #include #include - +#include namespace ProfileEvents { diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 262018f108c..71a9ba097c6 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -63,7 +63,7 @@ StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_f UInt64 offset = arguments.size() == 2 ? evaluateArgument(context, arguments[0]) : 0; UInt64 length = arguments.size() == 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); - auto res = std::make_shared(StorageID(getDatabaseName(), table_name), multithreaded, length, offset); + auto res = std::make_shared(StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset); res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionsGenerateSeries.cpp b/src/TableFunctions/TableFunctionsGenerateSeries.cpp new file mode 100644 index 00000000000..3941f1eadb2 --- /dev/null +++ b/src/TableFunctions/TableFunctionsGenerateSeries.cpp @@ -0,0 +1,100 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "registerTableFunctions.h" + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +namespace +{ + +/* numbers(limit), numbers_mt(limit) + * - the same as SELECT number FROM system.numbers LIMIT limit. + * Used for testing purposes, as a simple example of table function. + */ +class TableFunctionGenerateSeries : public ITableFunction +{ +public: + static constexpr auto name = "generate_series"; + std::string getName() const override { return name; } + bool hasStaticStructure() const override { return true; } +private: + StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns, bool is_insert_query) const override; + const char * getStorageTypeName() const override { return "SystemNumbers"; } + + UInt64 evaluateArgument(ContextPtr context, ASTPtr & argument) const; + + ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; +}; + +ColumnsDescription TableFunctionGenerateSeries::getActualTableStructure(ContextPtr /*context*/, bool /*is_insert_query*/) const +{ + /// NOTE: https://bugs.llvm.org/show_bug.cgi?id=47418 + return ColumnsDescription{{{"generate_series", std::make_shared()}}}; +} + +StoragePtr TableFunctionGenerateSeries::executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const +{ + if (const auto * function = ast_function->as()) + { + auto arguments = function->arguments->children; + + if (arguments.size() != 2 && arguments.size() != 3) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); + + UInt64 start = evaluateArgument(context, arguments[0]); + UInt64 stop = evaluateArgument(context, arguments[1]); + UInt64 interval = (arguments.size() == 3) ? evaluateArgument(context, arguments[2]) : UInt64{1}; + if (start > stop) { + auto res = std::make_shared(StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0); + res->startup(); + return res; + } + + auto res = std::make_shared(StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, (stop - start) / interval + 1, start, interval); + res->startup(); + return res; + } + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'limit' or 'offset, limit'.", getName()); +} + +UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr & argument) const +{ + const auto & [field, type] = evaluateConstantExpression(argument, context); + + if (!isNativeNumber(type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} expression, must be numeric type", type->getName()); + + Field converted = convertFieldToType(field, DataTypeUInt64()); + if (converted.isNull()) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The value {} is not representable as UInt64", + applyVisitor(FieldVisitorToString(), field)); + + return converted.safeGet(); +} + + +} + +void registerTableFunctionGenerateSeries(TableFunctionFactory & factory) +{ + factory.registerFunction({.documentation = {}, .allow_readonly = true}); + // factory.registerFunction({.documentation = {}, .allow_readonly = true}); +} + +} diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index 8c18c298f45..1631fa8e879 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -11,6 +11,7 @@ void registerTableFunctions() registerTableFunctionMerge(factory); registerTableFunctionRemote(factory); registerTableFunctionNumbers(factory); + registerTableFunctionGenerateSeries(factory); registerTableFunctionNull(factory); registerTableFunctionZeros(factory); registerTableFunctionExecutable(factory); diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index fae763e7dc8..111fbe8c22f 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -8,6 +8,7 @@ class TableFunctionFactory; void registerTableFunctionMerge(TableFunctionFactory & factory); void registerTableFunctionRemote(TableFunctionFactory & factory); void registerTableFunctionNumbers(TableFunctionFactory & factory); +void registerTableFunctionGenerateSeries(TableFunctionFactory & factory); void registerTableFunctionNull(TableFunctionFactory & factory); void registerTableFunctionZeros(TableFunctionFactory & factory); void registerTableFunctionExecutable(TableFunctionFactory & factory); From 7231e12e36069a2f7fb1c868d3ca2e856df17af4 Mon Sep 17 00:00:00 2001 From: zhongyuankai <872237106@qq.com> Date: Tue, 30 Jan 2024 21:47:16 +0800 Subject: [PATCH 0016/1165] fix build check --- src/Interpreters/InterpreterDropQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 6e0fda610ad..1fdbf2fa376 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -487,9 +487,9 @@ void InterpreterDropQuery::extendQueryLogElemImpl(DB::QueryLogElement & elem, co if (drop.database_and_tables) { auto & list = drop.database_and_tables->as(); - for (auto it = list.children.begin(); it != list.children.end(); ++it) + for (auto & child : list.children) { - auto identifier = dynamic_pointer_cast(*it); + auto identifier = dynamic_pointer_cast(child); if (identifier->name_parts.size() == 2) { auto quoted_database = backQuoteIfNeed(identifier->name_parts[0]); From 3f0cfbd8c0816b007ff85b1a3997696ce5ed3214 Mon Sep 17 00:00:00 2001 From: Daniil Ivanik Date: Sat, 3 Feb 2024 19:46:00 +0100 Subject: [PATCH 0017/1165] Kek --- src/Common/iota.cpp | 29 ++ src/Common/iota.h | 9 + .../QueryPlan/ReadFromSystemNumbersStep.cpp | 281 ++++++++++-------- 3 files changed, 197 insertions(+), 122 deletions(-) diff --git a/src/Common/iota.cpp b/src/Common/iota.cpp index 98f18eb195b..532c4bde76d 100644 --- a/src/Common/iota.cpp +++ b/src/Common/iota.cpp @@ -27,10 +27,39 @@ void iota(T * begin, size_t count, T first_value) return iotaImpl(begin, count, first_value); } +MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_HEADER(template void NO_INLINE), + iotaWithStepImpl, MULTITARGET_FUNCTION_BODY((T * begin, size_t count, T first_value, T step) /// NOLINT + { + for (size_t i = 0; i < count; i++) + *(begin + i) = static_cast(first_value + i * step); + }) +) + +template +void iota_with_step(T * begin, size_t count, T first_value, T step) +{ +#if USE_MULTITARGET_CODE + if (isArchSupported(TargetArch::AVX2)) + return iotaWithStepImplAVX2(begin, count, first_value, step); + + if (isArchSupported(TargetArch::SSE42)) + return iotaWithStepImplSSE42(begin, count, first_value, step); +#endif + return iotaWithStepImpl(begin, count, first_value, step); +} + template void iota(UInt8 * begin, size_t count, UInt8 first_value); template void iota(UInt32 * begin, size_t count, UInt32 first_value); template void iota(UInt64 * begin, size_t count, UInt64 first_value); #if defined(OS_DARWIN) template void iota(size_t * begin, size_t count, size_t first_value); #endif + +template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); +template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); +template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); +#if defined(OS_DARWIN) +extern template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); +#endif } diff --git a/src/Common/iota.h b/src/Common/iota.h index 7910274d15d..f40cde9d5db 100644 --- a/src/Common/iota.h +++ b/src/Common/iota.h @@ -31,4 +31,13 @@ extern template void iota(UInt64 * begin, size_t count, UInt64 first_value); #if defined(OS_DARWIN) extern template void iota(size_t * begin, size_t count, size_t first_value); #endif + +template void iota_with_step(T * begin, size_t count, T first_value, T step); + +extern template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); +extern template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); +extern template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); +#if defined(OS_DARWIN) +extern template void iota(size_t * begin, size_t count, size_t first_value, size_t step); +#endif } diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 13a14ffb917..f85473e43c3 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -12,6 +12,8 @@ #include #include +#include + namespace DB { @@ -30,9 +32,9 @@ public: : ISource(createHeader(column_name)), block_size(block_size_), next(offset_), step(step_), inner_step(inner_step_), inner_remainder(offset_ % inner_step_) { } - String getName() const override { return "Numbers"; } + static Block createHeader(const std::string& column_name) { return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), column_name)}; } protected: @@ -57,10 +59,7 @@ protected: ColumnUInt64::Container & vec = column->getData(); UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * end = &vec[filtered_block_size]; - iota(pos, static_cast(end - pos), UInt64{0}); - for (UInt64 p = 0; p < filtered_block_size; p += 1) { - vec[p] = vec[p] * inner_step + first_element; - } + iota_with_step(pos, static_cast(end - pos), first_element, inner_step); next += step; @@ -77,28 +76,46 @@ private: UInt64 inner_remainder; }; - -[[maybe_unused]] UInt128 sizeOfRange(const Range & r) -{ - UInt128 size; - if (r.right.isPositiveInfinity()) - return static_cast(std::numeric_limits::max()) - r.left.get() + r.left_included; - - size = static_cast(r.right.get()) - r.left.get() + 1; - - if (!r.left_included) - size--; - - if (!r.right_included) - size--; - assert(size >= 0); - return size; +struct RangeWithStep { + Range range; + UInt64 step; }; -[[maybe_unused]] auto sizeOfRanges(const Ranges & rs) +using RangesWithStep = std::vector; + +std::optional stepped_range_from_range(const Range& r, UInt64 step, UInt64 remainder) { + UInt64 begin = (r.left.get() / step) * step; + if (begin > std::numeric_limits::max() - remainder) { + return std::nullopt; + } + begin += remainder; + while (begin <= r.left.get() - r.left_included) { + if (std::numeric_limits::max() - step < begin) { + return std::nullopt; + } + begin += step; + } + + LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); + UInt128 right_edge = (r.right.get() + r.right_included); + if (begin >= right_edge) { + return std::nullopt; + } + return std::optional{RangeWithStep{Range(begin, true, static_cast(right_edge - 1), true), step}}; +} + +[[maybe_unused]] UInt128 sizeOfRange(const RangeWithStep & r) +{ + if (r.range.right.isPositiveInfinity()) + return static_cast(std::numeric_limits::max() - r.range.left.get()) / r.step + r.range.left_included; + + return static_cast(r.range.right.get() - r.range.left.get()) / r.step + 1; +}; + +[[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) { UInt128 total_size{}; - for (const Range & r : rs) + for (const RangeWithStep & r : rs) { /// total_size will never overflow total_size += sizeOfRange(r); @@ -127,7 +144,7 @@ public: using RangesStatePtr = std::shared_ptr; - [[maybe_unused]] NumbersRangedSource(const Ranges & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, const std::string& column_name) + [[maybe_unused]] NumbersRangedSource(const RangesWithStep & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, const std::string& column_name) : ISource(NumbersSource::createHeader(column_name)), ranges(ranges_), ranges_state(ranges_state_), base_block_size(base_block_size_) { } @@ -187,9 +204,9 @@ protected: if (ranges.empty()) return {}; - auto first_value = [](const Range & r) { return r.left.get() + (r.left_included ? 0 : 1); }; + auto first_value = [](const RangeWithStep & r) { return r.range.left.get() + (r.range.left_included ? 0 : 1); }; - auto last_value = [](const Range & r) { return r.right.get() - (r.right_included ? 0 : 1); }; + auto last_value = [](const RangeWithStep & r) { return r.range.right.get() - (r.range.right_included ? 0 : 1); }; /// Find the data range. /// If data left is small, shrink block size. @@ -215,31 +232,33 @@ protected: UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges ? end.offset_in_range - cursor.offset_in_range - : static_cast(last_value(range)) - first_value(range) + 1 - cursor.offset_in_range; + : static_cast(last_value(range) - first_value(range)) / range.step + 1 - cursor.offset_in_range; /// set value to block - auto set_value = [&pos](UInt128 & start_value, UInt128 & end_value) + auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) { if (end_value > std::numeric_limits::max()) { - while (start_value < end_value) - *(pos++) = start_value++; + while (start_value < end_value) { + *(pos++) = start_value; + start_value += this->step; + } } else { auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = end_value_64 - start_value_64; - iota(pos, static_cast(size), start_value_64); + iota_with_step(pos, static_cast(size), start_value_64, step); pos += size; } }; if (can_provide > need) { - UInt64 start_value = first_value(range) + cursor.offset_in_range; + UInt64 start_value = first_value(range) + cursor.offset_in_range * step; /// end_value will never overflow - iota(pos, static_cast(need), start_value); + iota_with_step(pos, static_cast(need), start_value, step); pos += need; provided += need; @@ -248,8 +267,8 @@ protected: else if (can_provide == need) { /// to avoid UInt64 overflow - UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range; - UInt128 end_value = start_value + need; + UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range * step; + UInt128 end_value = start_value + need * step; set_value(start_value, end_value); provided += need; @@ -259,8 +278,8 @@ protected: else { /// to avoid UInt64 overflow - UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range; - UInt128 end_value = start_value + can_provide; + UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range * step; + UInt128 end_value = start_value + can_provide * step; set_value(start_value, end_value); provided += static_cast(can_provide); @@ -277,13 +296,15 @@ protected: private: /// The ranges is shared between all streams. - Ranges ranges; + RangesWithStep ranges; /// Ranges state shared between all streams, actually is the start of the ranges. RangesStatePtr ranges_state; /// Base block size, will shrink when data left is not enough. UInt64 base_block_size; + + UInt64 step; }; } @@ -304,7 +325,7 @@ namespace /// Shrink ranges to size. /// For example: ranges: [1, 5], [8, 100]; size: 7, we will get [1, 5], [8, 9] -[[maybe_unused]] void shrinkRanges(Ranges & ranges, size_t size) +[[maybe_unused]] void shrinkRanges(RangesWithStep & ranges, size_t size) { size_t last_range_idx = 0; for (size_t i = 0; i < ranges.size(); i++) @@ -323,9 +344,9 @@ namespace else { auto & range = ranges[i]; - UInt64 right = range.left.get() + static_cast(size); - range.right = Field(right); - range.right_included = !range.left_included; + UInt64 right = range.range.left.get() + static_cast(size); + range.range.right = Field(right); + range.range.right_included = !range.range.left_included; last_range_idx = i; break; } @@ -393,101 +414,117 @@ Pipe ReadFromSystemNumbersStep::makePipe() num_streams = 1; /// Build rpn of query filters - // KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); + KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); Pipe pipe; Ranges ranges; - // if (condition.extractPlainRanges(ranges)) - // { - // /// Intersect ranges with table range - // std::optional table_range; - // std::optional overflowed_table_range; + if (condition.extractPlainRanges(ranges)) + { + LOG_DEBUG(&Poco::Logger::get("My logger"), "Use optimization"); + /// Intersect ranges with table range + std::optional table_range; + std::optional overflowed_table_range; - // if (numbers_storage.limit.has_value()) - // { - // if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) - // { - // table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); - // } - // /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) - // else - // { - // table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); - // auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); - // overflowed_table_range.emplace( - // FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); - // } - // } - // else - // { - // table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); - // } + if (numbers_storage.limit.has_value()) + { + if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) + { + table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); + } + /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) + else + { + table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); + auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); + overflowed_table_range.emplace( + FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); + } + } + else + { + table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); + } + LOG_DEBUG(&Poco::Logger::get("My logger"), "Found table ranges"); - // Ranges intersected_ranges; - // for (auto & r : ranges) - // { - // auto intersected_range = table_range->intersectWith(r); - // if (intersected_range) - // intersected_ranges.push_back(*intersected_range); - // } - // /// intersection with overflowed_table_range goes back. - // if (overflowed_table_range.has_value()) - // { - // for (auto & r : ranges) - // { - // auto intersected_range = overflowed_table_range->intersectWith(r); - // if (intersected_range) - // intersected_ranges.push_back(*overflowed_table_range); - // } - // } + RangesWithStep intersected_ranges; + for (auto & r : ranges) + { + auto intersected_range = table_range->intersectWith(r); + if (intersected_range.has_value()) { + auto range_with_step = stepped_range_from_range(intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); + if (range_with_step.has_value()) { + intersected_ranges.push_back(*range_with_step); + } + } + } - // /// ranges is blank, return a source who has no data - // if (intersected_ranges.empty()) - // { - // pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); - // return pipe; - // } - // const auto & limit_length = limit_length_and_offset.first; - // const auto & limit_offset = limit_length_and_offset.second; - // /// If intersected ranges is limited or we can pushdown limit. - // if (!intersected_ranges.rbegin()->right.isPositiveInfinity() || should_pushdown_limit) - // { - // UInt128 total_size = sizeOfRanges(intersected_ranges); - // UInt128 query_limit = limit_length + limit_offset; + for (const auto& range : intersected_ranges) { + LOG_DEBUG(&Poco::Logger::get("Ranges"), "Left: {}; Right {}, LI: {}, RI: {}, Step: {}", range.range.left.get(), range.range.right.get(), range.range.left_included, range.range.right_included, range.step); + // std::cout << + } + /// intersection with overflowed_table_range goes back. + if (overflowed_table_range.has_value()) + { + for (auto & r : ranges) + { + auto intersected_range = overflowed_table_range->intersectWith(r); + if (intersected_range) { + auto range_with_step = stepped_range_from_range(intersected_range.value(), numbers_storage.step, static_cast((static_cast(numbers_storage.offset) + std::numeric_limits::max() + 1) % numbers_storage.step)); + if (range_with_step) { + intersected_ranges.push_back(*range_with_step); + } + } + } + } - // /// limit total_size by query_limit - // if (should_pushdown_limit && query_limit < total_size) - // { - // total_size = query_limit; - // /// We should shrink intersected_ranges for case: - // /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 - // shrinkRanges(intersected_ranges, total_size); - // } + /// ranges is blank, return a source who has no data + if (intersected_ranges.empty()) + { + pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); + return pipe; + } + const auto & limit_length = limit_length_and_offset.first; + const auto & limit_offset = limit_length_and_offset.second; - // checkLimits(size_t(total_size)); + /// If intersected ranges is limited or we can pushdown limit. + if (!intersected_ranges.rbegin()->range.right.isPositiveInfinity() || should_pushdown_limit) + { + UInt128 total_size = sizeOfRanges(intersected_ranges); + UInt128 query_limit = limit_length + limit_offset; - // if (total_size / max_block_size < num_streams) - // num_streams = static_cast(total_size / max_block_size); + /// limit total_size by query_limit + if (should_pushdown_limit && query_limit < total_size) + { + total_size = query_limit; + /// We should shrink intersected_ranges for case: + /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 + shrinkRanges(intersected_ranges, total_size); + } - // if (num_streams == 0) - // num_streams = 1; + checkLimits(size_t(total_size)); - // /// Ranges state, all streams will share the state. - // auto ranges_state = std::make_shared(); - // for (size_t i = 0; i < num_streams; ++i) - // { - // auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size, numbers_storage.column_name); + if (total_size / max_block_size < num_streams) + num_streams = static_cast(total_size / max_block_size); - // if (i == 0) - // source->addTotalRowsApprox(total_size); + if (num_streams == 0) + num_streams = 1; - // pipe.addSource(std::move(source)); - // } - // return pipe; - // } - // } + /// Ranges state, all streams will share the state. + auto ranges_state = std::make_shared(); + for (size_t i = 0; i < num_streams; ++i) + { + auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size, numbers_storage.column_name); + + if (i == 0) + source->addTotalRowsApprox(total_size); + + pipe.addSource(std::move(source)); + } + return pipe; + } + } /// Fall back to NumbersSource for (size_t i = 0; i < num_streams; ++i) From 623b42574587073845a76a5e28a502a792ee6662 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 6 Feb 2024 21:34:09 +0000 Subject: [PATCH 0018/1165] Add feature with the right author name --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 175 +++++++++++++----- src/Storages/SelectQueryInfo.h | 2 +- src/Storages/System/StorageSystemNumbers.cpp | 8 +- src/Storages/System/StorageSystemNumbers.h | 15 +- src/Storages/System/attachSystemTables.cpp | 2 +- src/TableFunctions/CMakeLists.txt | 2 +- src/TableFunctions/ITableFunction.cpp | 2 +- ...es.cpp => TableFunctionGenerateSeries.cpp} | 55 ++++-- src/TableFunctions/TableFunctionNumbers.cpp | 45 +++-- .../02970_generate_series.reference | 28 +++ .../0_stateless/02970_generate_series.sql | 14 ++ 11 files changed, 254 insertions(+), 94 deletions(-) rename src/TableFunctions/{TableFunctionsGenerateSeries.cpp => TableFunctionGenerateSeries.cpp} (65%) create mode 100644 tests/queries/0_stateless/02970_generate_series.reference create mode 100644 tests/queries/0_stateless/02970_generate_series.sql diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index f85473e43c3..4b957778c43 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -28,32 +28,37 @@ namespace class NumbersSource : public ISource { public: - NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_, const std::string& column_name, UInt64 inner_step_) - : ISource(createHeader(column_name)), block_size(block_size_), next(offset_), step(step_), inner_step(inner_step_), inner_remainder(offset_ % inner_step_) + NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_, const std::string & column_name, UInt64 inner_step_) + : ISource(createHeader(column_name)) + , block_size(block_size_) + , next(offset_) + , step(step_) + , inner_step(inner_step_) + , inner_remainder(offset_ % inner_step_) { } String getName() const override { return "Numbers"; } - - static Block createHeader(const std::string& column_name) { return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), column_name)}; } + static Block createHeader(const std::string & column_name) + { + return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), column_name)}; + } protected: Chunk generate() override { - UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. UInt64 first_element = (curr / inner_step) * inner_step + inner_remainder; - if (first_element < curr) { + if (first_element < curr) first_element += inner_step; - } UInt64 filtered_block_size = 0; - if (first_element - curr >= block_size) { + if (first_element - curr >= block_size) + { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, filtered_block_size}; } - if (first_element - curr < block_size) { + if (first_element - curr < block_size) filtered_block_size = (block_size - (first_element - curr) - 1) / inner_step + 1; - } auto column = ColumnUInt64::create(filtered_block_size); ColumnUInt64::Container & vec = column->getData(); @@ -76,32 +81,37 @@ private: UInt64 inner_remainder; }; -struct RangeWithStep { +struct RangeWithStep +{ Range range; UInt64 step; }; using RangesWithStep = std::vector; -std::optional stepped_range_from_range(const Range& r, UInt64 step, UInt64 remainder) { - UInt64 begin = (r.left.get() / step) * step; - if (begin > std::numeric_limits::max() - remainder) { +std::optional stepped_range_from_range(const Range & r, UInt64 step, UInt64 remainder) +{ + if ((r.right.get() == 0) && (!r.right_included)) + return std::nullopt; + UInt64 begin = (r.left.get() / step) * step; + if (begin > std::numeric_limits::max() - remainder) return std::nullopt; - } begin += remainder; - while (begin <= r.left.get() - r.left_included) { - if (std::numeric_limits::max() - step < begin) { + + // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); + // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); + while ((r.left_included <= r.left.get()) && (begin <= r.left.get() - r.left_included)) + { + if (std::numeric_limits::max() - step < begin) return std::nullopt; - } begin += step; } - LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); - UInt128 right_edge = (r.right.get() + r.right_included); - if (begin >= right_edge) { + // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); + if ((begin >= r.right_included) && (begin - r.right_included >= r.right.get())) return std::nullopt; - } - return std::optional{RangeWithStep{Range(begin, true, static_cast(right_edge - 1), true), step}}; + UInt64 right_edge_included = r.right.get() - (1 - r.right_included); + return std::optional{RangeWithStep{Range(begin, true, right_edge_included, true), step}}; } [[maybe_unused]] UInt128 sizeOfRange(const RangeWithStep & r) @@ -144,8 +154,17 @@ public: using RangesStatePtr = std::shared_ptr; - [[maybe_unused]] NumbersRangedSource(const RangesWithStep & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, const std::string& column_name) - : ISource(NumbersSource::createHeader(column_name)), ranges(ranges_), ranges_state(ranges_state_), base_block_size(base_block_size_) + [[maybe_unused]] NumbersRangedSource( + const RangesWithStep & ranges_, + RangesStatePtr & ranges_state_, + UInt64 base_block_size_, + UInt64 step_, + const std::string & column_name) + : ISource(NumbersSource::createHeader(column_name)) + , ranges(ranges_) + , ranges_state(ranges_state_) + , base_block_size(base_block_size_) + , step(step_) { } @@ -158,6 +177,7 @@ protected: { std::lock_guard lock(ranges_state->mutex); + UInt64 need = base_block_size_; UInt64 size = 0; /// how many item found. @@ -196,6 +216,10 @@ protected: } ranges_state->pos = end; + + LOG_DEBUG(&Poco::Logger::get("Range borders"), "Begin: {} {}", start.offset_in_ranges, static_cast(start.offset_in_range)); + LOG_DEBUG(&Poco::Logger::get("Range borders"), "End: {} {}", end.offset_in_ranges, static_cast(end.offset_in_range)); + return size; } @@ -234,12 +258,19 @@ protected: ? end.offset_in_range - cursor.offset_in_range : static_cast(last_value(range) - first_value(range)) / range.step + 1 - cursor.offset_in_range; + LOG_DEBUG( + &Poco::Logger::get("Generate"), + "Can Provide: {}, Block size: {}", + static_cast(can_provide), + static_cast(block_size)); + /// set value to block auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) { if (end_value > std::numeric_limits::max()) { - while (start_value < end_value) { + while (start_value < end_value) + { *(pos++) = start_value; start_value += this->step; } @@ -248,7 +279,9 @@ protected: { auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); - auto size = end_value_64 - start_value_64; + auto size = (end_value_64 - start_value_64) / this->step; + LOG_DEBUG( + &Poco::Logger::get("Iota"), "Size: {}, Step: {}, Start: {}", static_cast(size), this->step, start_value_64); iota_with_step(pos, static_cast(size), start_value_64, step); pos += size; } @@ -374,7 +407,7 @@ ReadFromSystemNumbersStep::ReadFromSystemNumbersStep( , key_expression{KeyDescription::parse(column_names[0], storage_snapshot->metadata->columns, context).expression} , max_block_size{max_block_size_} , num_streams{num_streams_} - , limit_length_and_offset(InterpreterSelectQuery::getLimitLengthAndOffset(query_info.query->as(), context)) + , limit_length_and_offset(InterpreterSelectQuery::getLimitLengthAndOffset(query_info.query->as(), context)) , should_pushdown_limit(shouldPushdownLimit(query_info, limit_length_and_offset.first)) , limit(query_info.limit) , storage_limits(query_info.storage_limits) @@ -410,14 +443,28 @@ Pipe ReadFromSystemNumbersStep::makePipe() { auto & numbers_storage = storage->as(); + LOG_DEBUG( + &Poco::Logger::get("Parameters"), + "Parameters: Limit: {}, Offset: {} Step: {}", + numbers_storage.limit.value(), + numbers_storage.offset, + numbers_storage.step); + if (!numbers_storage.multithreaded) num_streams = 1; + Pipe pipe; + Ranges ranges; + + if (numbers_storage.limit.has_value() && (numbers_storage.limit.value() == 0)) + { + pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); + return pipe; + } + /// Build rpn of query filters KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); - Pipe pipe; - Ranges ranges; if (condition.extractPlainRanges(ranges)) { @@ -430,14 +477,15 @@ Pipe ReadFromSystemNumbersStep::makePipe() { if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) { - table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); + table_range.emplace( + FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); } /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) else { table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); - overflowed_table_range.emplace( + overflowed_table_range.emplace( FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); } } @@ -451,34 +499,59 @@ Pipe ReadFromSystemNumbersStep::makePipe() for (auto & r : ranges) { auto intersected_range = table_range->intersectWith(r); - if (intersected_range.has_value()) { - auto range_with_step = stepped_range_from_range(intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); - if (range_with_step.has_value()) { + if (intersected_range.has_value()) + { + LOG_DEBUG( + &Poco::Logger::get("Ranges"), + "Ranges: {} {} {} {}", + intersected_range->left.get(), + intersected_range->right.get(), + intersected_range->left_included, + intersected_range->right_included); + auto range_with_step = stepped_range_from_range( + intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); + if (range_with_step.has_value()) + { + LOG_DEBUG( + &Poco::Logger::get("Ranges With Step"), + "Ranges: {} {} {} {} {}", + range_with_step->range.left.get(), + range_with_step->range.right.get(), + range_with_step->range.left_included, + range_with_step->range.right_included, + range_with_step->step); intersected_ranges.push_back(*range_with_step); } } } - for (const auto& range : intersected_ranges) { - LOG_DEBUG(&Poco::Logger::get("Ranges"), "Left: {}; Right {}, LI: {}, RI: {}, Step: {}", range.range.left.get(), range.range.right.get(), range.range.left_included, range.range.right_included, range.step); - // std::cout << - } /// intersection with overflowed_table_range goes back. if (overflowed_table_range.has_value()) { for (auto & r : ranges) { auto intersected_range = overflowed_table_range->intersectWith(r); - if (intersected_range) { - auto range_with_step = stepped_range_from_range(intersected_range.value(), numbers_storage.step, static_cast((static_cast(numbers_storage.offset) + std::numeric_limits::max() + 1) % numbers_storage.step)); - if (range_with_step) { + if (intersected_range) + { + auto range_with_step = stepped_range_from_range( + intersected_range.value(), + numbers_storage.step, + static_cast( + (static_cast(numbers_storage.offset) + std::numeric_limits::max() + 1) + % numbers_storage.step)); + if (range_with_step) intersected_ranges.push_back(*range_with_step); - } } } } + // for (const auto& range : intersected_ranges) + // { + // LOG_DEBUG(&Poco::Logger::get("Ranges with step"), "Left: {}; Right {}, LI: {}, RI: {}, Step: {}", range.range.left.get(), range.range.right.get(), range.range.left_included, range.range.right_included, range.step); + // // std::cout << + // } + /// ranges is blank, return a source who has no data if (intersected_ranges.empty()) { @@ -492,6 +565,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() if (!intersected_ranges.rbegin()->range.right.isPositiveInfinity() || should_pushdown_limit) { UInt128 total_size = sizeOfRanges(intersected_ranges); + LOG_DEBUG(&Poco::Logger::get("Total_Size"), "Total Size: {}", static_cast(total_size)); UInt128 query_limit = limit_length + limit_offset; /// limit total_size by query_limit @@ -515,7 +589,8 @@ Pipe ReadFromSystemNumbersStep::makePipe() auto ranges_state = std::make_shared(); for (size_t i = 0; i < num_streams; ++i) { - auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size, numbers_storage.column_name); + auto source = std::make_shared( + intersected_ranges, ranges_state, max_block_size, numbers_storage.step, numbers_storage.column_name); if (i == 0) source->addTotalRowsApprox(total_size); @@ -529,12 +604,16 @@ Pipe ReadFromSystemNumbersStep::makePipe() /// Fall back to NumbersSource for (size_t i = 0; i < num_streams; ++i) { - auto source - = std::make_shared(max_block_size, numbers_storage.offset + i * max_block_size, num_streams * max_block_size, numbers_storage.column_name, numbers_storage.step); + auto source = std::make_shared( + max_block_size, + numbers_storage.offset + i * max_block_size, + num_streams * max_block_size, + numbers_storage.column_name, + numbers_storage.step); if (numbers_storage.limit && i == 0) { - auto rows_appr = *(numbers_storage.limit); + auto rows_appr = (*numbers_storage.limit - 1) / numbers_storage.step + 1; if (limit > 0 && limit < rows_appr) rows_appr = limit; source->addTotalRowsApprox(rows_appr); @@ -546,7 +625,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() if (numbers_storage.limit) { size_t i = 0; - auto storage_limit = *(numbers_storage.limit); + auto storage_limit = (*numbers_storage.limit - 1) / numbers_storage.step + 1; /// This formula is how to split 'limit' elements to 'num_streams' chunks almost uniformly. pipe.addSimpleTransform( [&](const Block & header) diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 2b4afaa6345..662a5c0ef5a 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -229,4 +229,4 @@ struct SelectQueryInfo bool isFinal() const; }; - } +} diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index cd7207917a9..4c319ec7105 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -16,7 +16,13 @@ namespace DB { -StorageSystemNumbers::StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, const std::string& column_name_, std::optional limit_, UInt64 offset_, UInt64 step_) +StorageSystemNumbers::StorageSystemNumbers( + const StorageID & table_id, + bool multithreaded_, + const std::string & column_name_, + std::optional limit_, + UInt64 offset_, + UInt64 step_) : IStorage(table_id), multithreaded(multithreaded_), limit(limit_), offset(offset_), column_name(column_name_), step(step_) { StorageInMemoryMetadata storage_metadata; diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index ffe87b8ad14..9663ee25251 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -10,11 +10,17 @@ namespace DB class Context; -class StorageSystemNumbers final : public IStorage +class StorageSystemNumbers final : public IStorage { public: /// Otherwise, streams concurrently increment atomic. - StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, const std::string& column_name, std::optional limit_ = std::nullopt, UInt64 offset_ = 0, UInt64 step_ = 1); + StorageSystemNumbers( + const StorageID & table_id, + bool multithreaded_, + const std::string & column_name, + std::optional limit_ = std::nullopt, + UInt64 offset_ = 0, + UInt64 step_ = 1); std::string getName() const override { return "SystemNumbers"; } @@ -30,7 +36,6 @@ public: bool hasEvenlyDistributedRead() const override { return true; } bool isSystemStorage() const override { return true; } - bool supportsTransactions() const override { return true; } private: @@ -38,11 +43,9 @@ private: bool multithreaded; std::optional limit; - UInt64 offset; + UInt64 offset;` std::string column_name; - UInt64 step; - }; } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index ddd89709b6a..9eacb07bd8d 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -120,7 +120,7 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b attach(context, system_database, "one", "This table contains a single row with a single dummy UInt8 column containing the value 0. Used when the table is not specified explicitly, for example in queries like `SELECT 1`."); attach(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "number"); attach(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "number"); - // attach(context, system_database, "generate_series", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", false, "generate_series"); + attach(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); attach(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attach(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); attach(context, system_database, "databases", "Lists all databases of the current server."); diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index c5c2a660935..770990cc405 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -29,7 +29,7 @@ if (TARGET ch_contrib::azure_sdk) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::azure_sdk) endif () -if (TARGET ch_co`trib::simdjson) +if (TARGET ch_contrib::simdjson) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::simdjson) endif () diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index c854b6b0f9c..137e1dc27fe 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -5,7 +5,7 @@ #include #include #include -#include + namespace ProfileEvents { diff --git a/src/TableFunctions/TableFunctionsGenerateSeries.cpp b/src/TableFunctions/TableFunctionGenerateSeries.cpp similarity index 65% rename from src/TableFunctions/TableFunctionsGenerateSeries.cpp rename to src/TableFunctions/TableFunctionGenerateSeries.cpp index 3941f1eadb2..88d7b0d1a71 100644 --- a/src/TableFunctions/TableFunctionsGenerateSeries.cpp +++ b/src/TableFunctions/TableFunctionGenerateSeries.cpp @@ -1,13 +1,13 @@ +#include +#include +#include +#include +#include +#include #include #include -#include -#include #include -#include -#include -#include -#include -#include +#include #include "registerTableFunctions.h" @@ -18,6 +18,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INVALID_SETTING_VALUE; } namespace @@ -33,8 +34,14 @@ public: static constexpr auto name = "generate_series"; std::string getName() const override { return name; } bool hasStaticStructure() const override { return true; } + private: - StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns, bool is_insert_query) const override; + StoragePtr executeImpl( + const ASTPtr & ast_function, + ContextPtr context, + const std::string & table_name, + ColumnsDescription cached_columns, + bool is_insert_query) const override; const char * getStorageTypeName() const override { return "SystemNumbers"; } UInt64 evaluateArgument(ContextPtr context, ASTPtr & argument) const; @@ -48,25 +55,31 @@ ColumnsDescription TableFunctionGenerateSeries::getActualTableStructure(ContextP return ColumnsDescription{{{"generate_series", std::make_shared()}}}; } -StoragePtr TableFunctionGenerateSeries::executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const +StoragePtr TableFunctionGenerateSeries::executeImpl( + const ASTPtr & ast_function, + ContextPtr context, + const std::string & table_name, + ColumnsDescription /*cached_columns*/, + bool /*is_insert_query*/) const { if (const auto * function = ast_function->as()) { auto arguments = function->arguments->children; if (arguments.size() != 2 && arguments.size() != 3) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); UInt64 start = evaluateArgument(context, arguments[0]); UInt64 stop = evaluateArgument(context, arguments[1]); - UInt64 interval = (arguments.size() == 3) ? evaluateArgument(context, arguments[2]) : UInt64{1}; - if (start > stop) { - auto res = std::make_shared(StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0); - res->startup(); - return res; - } - - auto res = std::make_shared(StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, (stop - start) / interval + 1, start, interval); + UInt64 step = (arguments.size() == 3) ? evaluateArgument(context, arguments[2]) : UInt64{1}; + if (step == UInt64{0}) + throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Table function '{}' requires step to be a positive number", getName()); + auto res = (start > stop) + ? std::make_shared( + StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0, 0, 0) + : std::make_shared( + StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, (stop - start) + 1, start, step); res->startup(); return res; } @@ -82,8 +95,10 @@ UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr Field converted = convertFieldToType(field, DataTypeUInt64()); if (converted.isNull()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The value {} is not representable as UInt64", - applyVisitor(FieldVisitorToString(), field)); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The value {} is not representable as UInt64", + applyVisitor(FieldVisitorToString(), field)); return converted.safeGet(); } diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 71a9ba097c6..bcda8dc6a5e 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -1,13 +1,13 @@ +#include +#include +#include +#include +#include +#include #include #include -#include -#include #include -#include -#include -#include -#include -#include +#include #include "registerTableFunctions.h" @@ -16,8 +16,8 @@ namespace DB namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; } namespace @@ -34,8 +34,14 @@ public: static constexpr auto name = multithreaded ? "numbers_mt" : "numbers"; std::string getName() const override { return name; } bool hasStaticStructure() const override { return true; } + private: - StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns, bool is_insert_query) const override; + StoragePtr executeImpl( + const ASTPtr & ast_function, + ContextPtr context, + const std::string & table_name, + ColumnsDescription cached_columns, + bool is_insert_query) const override; const char * getStorageTypeName() const override { return "SystemNumbers"; } UInt64 evaluateArgument(ContextPtr context, ASTPtr & argument) const; @@ -51,19 +57,26 @@ ColumnsDescription TableFunctionNumbers::getActualTableStructure( } template -StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const +StoragePtr TableFunctionNumbers::executeImpl( + const ASTPtr & ast_function, + ContextPtr context, + const std::string & table_name, + ColumnsDescription /*cached_columns*/, + bool /*is_insert_query*/) const { if (const auto * function = ast_function->as()) { auto arguments = function->arguments->children; if (arguments.size() != 1 && arguments.size() != 2) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); UInt64 offset = arguments.size() == 2 ? evaluateArgument(context, arguments[0]) : 0; UInt64 length = arguments.size() == 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); - auto res = std::make_shared(StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset); + auto res = std::make_shared( + StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset); res->startup(); return res; } @@ -80,8 +93,10 @@ UInt64 TableFunctionNumbers::evaluateArgument(ContextPtr context, Field converted = convertFieldToType(field, DataTypeUInt64()); if (converted.isNull()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The value {} is not representable as UInt64", - applyVisitor(FieldVisitorToString(), field)); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The value {} is not representable as UInt64", + applyVisitor(FieldVisitorToString(), field)); return converted.safeGet(); } diff --git a/tests/queries/0_stateless/02970_generate_series.reference b/tests/queries/0_stateless/02970_generate_series.reference new file mode 100644 index 00000000000..9e6f1db911e --- /dev/null +++ b/tests/queries/0_stateless/02970_generate_series.reference @@ -0,0 +1,28 @@ +0 +1 +4 +8 +500000001 +50000000 +100000001 +0 +10 +13 +16 +19 +7 +17 +27 +37 +47 +57 +67 +77 +17 +22 +27 +32 +37 +42 +47 +52 diff --git a/tests/queries/0_stateless/02970_generate_series.sql b/tests/queries/0_stateless/02970_generate_series.sql new file mode 100644 index 00000000000..045f584a622 --- /dev/null +++ b/tests/queries/0_stateless/02970_generate_series.sql @@ -0,0 +1,14 @@ +SELECT count() FROM generate_series(5, 4); +SELECT count() FROM generate_series(0, 0); +SELECT count() FROM generate_series(10, 20, 3); +SELECT count() FROM generate_series(7, 77, 10); +SELECT count() FROM generate_series(0, 1000000000, 2); +SELECT count() FROM generate_series(0, 999999999, 20); +SELECT count() FROM generate_series(0, 1000000000, 2) WHERE generate_series % 5 == 0; + +SELECT * FROM generate_series(5, 4); +SELECT * FROM generate_series(0, 0); +SELECT * FROM generate_series(10, 20, 3); +SELECT * FROM generate_series(7, 77, 10); +SELECT * FROM generate_series(7, 52, 5) WHERE generate_series >= 13; + From 145e425ddd5707a5852dd3c6ac2672ccbd68e2bd Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 7 Feb 2024 15:29:45 +0000 Subject: [PATCH 0019/1165] Added Documentation --- .../table-functions/generate_series.md | 25 +++++++++ .../QueryPlan/ReadFromSystemNumbersStep.cpp | 53 ------------------- src/Storages/System/StorageSystemNumbers.h | 2 +- 3 files changed, 26 insertions(+), 54 deletions(-) create mode 100644 docs/en/sql-reference/table-functions/generate_series.md diff --git a/docs/en/sql-reference/table-functions/generate_series.md b/docs/en/sql-reference/table-functions/generate_series.md new file mode 100644 index 00000000000..de34e10ac76 --- /dev/null +++ b/docs/en/sql-reference/table-functions/generate_series.md @@ -0,0 +1,25 @@ +--- +slug: /en/sql-reference/table-functions/generate_series +sidebar_position: ? +sidebar_label: generate_series +--- + +# generate_series + +`generate_series(START, STOP)` - Returns a table with the single ‘generate_series’ column (UInt64) that contains integers from start to stop inclusively. + +`generate_series(START, STOP, STEP)` - Returns a table with the single ‘generate_series’ column (UInt64) that contains integers from start to stop inclusively with spacing between values given by STEP. + +The following queries return tables with the same content but different column names: + +``` sql +SELECT * FROM numbers(10, 5); +SELECT * FROM generate_series(10, 14); +``` + +And the following queries return tables with the same content but different column names (but the second option is more efficient): + +``` sql +SELECT * FROM numbers(10, 11) WHERE number % 3 == (10 % 3); +SELECT * FROM generate_series(10, 20, 3) ; +``` \ No newline at end of file diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 4b957778c43..3bb2e0cd69d 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -98,8 +98,6 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st return std::nullopt; begin += remainder; - // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); - // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); while ((r.left_included <= r.left.get()) && (begin <= r.left.get() - r.left_included)) { if (std::numeric_limits::max() - step < begin) @@ -107,7 +105,6 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st begin += step; } - // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); if ((begin >= r.right_included) && (begin - r.right_included >= r.right.get())) return std::nullopt; UInt64 right_edge_included = r.right.get() - (1 - r.right_included); @@ -217,9 +214,6 @@ protected: ranges_state->pos = end; - LOG_DEBUG(&Poco::Logger::get("Range borders"), "Begin: {} {}", start.offset_in_ranges, static_cast(start.offset_in_range)); - LOG_DEBUG(&Poco::Logger::get("Range borders"), "End: {} {}", end.offset_in_ranges, static_cast(end.offset_in_range)); - return size; } @@ -258,12 +252,6 @@ protected: ? end.offset_in_range - cursor.offset_in_range : static_cast(last_value(range) - first_value(range)) / range.step + 1 - cursor.offset_in_range; - LOG_DEBUG( - &Poco::Logger::get("Generate"), - "Can Provide: {}, Block size: {}", - static_cast(can_provide), - static_cast(block_size)); - /// set value to block auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) { @@ -280,8 +268,6 @@ protected: auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = (end_value_64 - start_value_64) / this->step; - LOG_DEBUG( - &Poco::Logger::get("Iota"), "Size: {}, Step: {}, Start: {}", static_cast(size), this->step, start_value_64); iota_with_step(pos, static_cast(size), start_value_64, step); pos += size; } @@ -443,13 +429,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() { auto & numbers_storage = storage->as(); - LOG_DEBUG( - &Poco::Logger::get("Parameters"), - "Parameters: Limit: {}, Offset: {} Step: {}", - numbers_storage.limit.value(), - numbers_storage.offset, - numbers_storage.step); - if (!numbers_storage.multithreaded) num_streams = 1; @@ -468,7 +447,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() if (condition.extractPlainRanges(ranges)) { - LOG_DEBUG(&Poco::Logger::get("My logger"), "Use optimization"); /// Intersect ranges with table range std::optional table_range; std::optional overflowed_table_range; @@ -493,36 +471,11 @@ Pipe ReadFromSystemNumbersStep::makePipe() { table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); } - LOG_DEBUG(&Poco::Logger::get("My logger"), "Found table ranges"); RangesWithStep intersected_ranges; for (auto & r : ranges) { auto intersected_range = table_range->intersectWith(r); - if (intersected_range.has_value()) - { - LOG_DEBUG( - &Poco::Logger::get("Ranges"), - "Ranges: {} {} {} {}", - intersected_range->left.get(), - intersected_range->right.get(), - intersected_range->left_included, - intersected_range->right_included); - auto range_with_step = stepped_range_from_range( - intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); - if (range_with_step.has_value()) - { - LOG_DEBUG( - &Poco::Logger::get("Ranges With Step"), - "Ranges: {} {} {} {} {}", - range_with_step->range.left.get(), - range_with_step->range.right.get(), - range_with_step->range.left_included, - range_with_step->range.right_included, - range_with_step->step); - intersected_ranges.push_back(*range_with_step); - } - } } @@ -546,11 +499,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() } } - // for (const auto& range : intersected_ranges) - // { - // LOG_DEBUG(&Poco::Logger::get("Ranges with step"), "Left: {}; Right {}, LI: {}, RI: {}, Step: {}", range.range.left.get(), range.range.right.get(), range.range.left_included, range.range.right_included, range.step); - // // std::cout << - // } /// ranges is blank, return a source who has no data if (intersected_ranges.empty()) @@ -565,7 +513,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() if (!intersected_ranges.rbegin()->range.right.isPositiveInfinity() || should_pushdown_limit) { UInt128 total_size = sizeOfRanges(intersected_ranges); - LOG_DEBUG(&Poco::Logger::get("Total_Size"), "Total Size: {}", static_cast(total_size)); UInt128 query_limit = limit_length + limit_offset; /// limit total_size by query_limit diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index 9663ee25251..298721984b8 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -43,7 +43,7 @@ private: bool multithreaded; std::optional limit; - UInt64 offset;` + UInt64 offset; std::string column_name; UInt64 step; }; From 03aaedace439f5db6d9a6aaf91a1b2f978b0f6a9 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 9 Feb 2024 12:05:01 +0000 Subject: [PATCH 0020/1165] Fix bug --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 3bb2e0cd69d..bc14547889b 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -435,6 +435,8 @@ Pipe ReadFromSystemNumbersStep::makePipe() Pipe pipe; Ranges ranges; + // LOG_DEBUG(&Poco::Logger::get("parameters"), "Parameters: {} {} {}", numbers_storage.step, numbers_storage.limit.value(), numbers_storage.offset); + if (numbers_storage.limit.has_value() && (numbers_storage.limit.value() == 0)) { pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); @@ -476,6 +478,15 @@ Pipe ReadFromSystemNumbersStep::makePipe() for (auto & r : ranges) { auto intersected_range = table_range->intersectWith(r); + if (intersected_range.has_value()) + { + auto range_with_step = stepped_range_from_range( + intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); + if (range_with_step.has_value()) + { + intersected_ranges.push_back(*range_with_step); + } + } } From 1b2f23247b7f115ba92b9908d224d4e78e8649f4 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 9 Feb 2024 12:28:54 +0000 Subject: [PATCH 0021/1165] Change documentation --- docs/en/sql-reference/table-functions/generate_series.md | 2 +- src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/en/sql-reference/table-functions/generate_series.md b/docs/en/sql-reference/table-functions/generate_series.md index de34e10ac76..c5d29369627 100644 --- a/docs/en/sql-reference/table-functions/generate_series.md +++ b/docs/en/sql-reference/table-functions/generate_series.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/table-functions/generate_series -sidebar_position: ? +sidebar_position: 146 sidebar_label: generate_series --- diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index bc14547889b..ab2f726aeb5 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -483,9 +483,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() auto range_with_step = stepped_range_from_range( intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); if (range_with_step.has_value()) - { intersected_ranges.push_back(*range_with_step); - } } } From 79f91003538a71014eb035dca024285f2fbba7d5 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 9 Feb 2024 14:17:25 +0000 Subject: [PATCH 0022/1165] To pull --- src/TableFunctions/TableFunctionGenerateSeries.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TableFunctions/TableFunctionGenerateSeries.cpp b/src/TableFunctions/TableFunctionGenerateSeries.cpp index 88d7b0d1a71..65c4c4915c2 100644 --- a/src/TableFunctions/TableFunctionGenerateSeries.cpp +++ b/src/TableFunctions/TableFunctionGenerateSeries.cpp @@ -77,7 +77,7 @@ StoragePtr TableFunctionGenerateSeries::executeImpl( throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Table function '{}' requires step to be a positive number", getName()); auto res = (start > stop) ? std::make_shared( - StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0, 0, 0) + StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0, 0, 1) : std::make_shared( StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, (stop - start) + 1, start, step); res->startup(); From f7dbcdd7e7e00d4fb6d30a02ebcb4a3befcd3190 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 9 Feb 2024 18:12:24 +0000 Subject: [PATCH 0023/1165] Made refactoring --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 41 ++++++++++++------- src/Storages/System/StorageSystemNumbers.h | 28 +++++++++++++ 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 3a905a56aa1..2488fa37643 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -9,10 +9,12 @@ #include #include #include +#include #include #include #include +#include "base/types.h" namespace DB { @@ -28,13 +30,13 @@ namespace class NumbersSource : public ISource { public: - NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_, const std::string & column_name, UInt64 inner_step_) + NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 chunk_step_, const std::string & column_name, UInt64 step_, UInt64 remainder_) : ISource(createHeader(column_name)) , block_size(block_size_) , next(offset_) + , chunk_step(chunk_step_) , step(step_) - , inner_step(inner_step_) - , inner_remainder(offset_ % inner_step_) + , remainder(remainder_) { } String getName() const override { return "Numbers"; } @@ -48,25 +50,33 @@ protected: Chunk generate() override { UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. - UInt64 first_element = (curr / inner_step) * inner_step + inner_remainder; - if (first_element < curr) - first_element += inner_step; - UInt64 filtered_block_size = 0; + UInt64 first_element = (curr / step) * step; + if (first_element > std::numeric_limits::max() - remainder) { + auto column = ColumnUInt64::create(0); + return {Columns{std::move(column)}, 0}; + } + first_element += remainder; + if (first_element < curr) { + if (first_element > std::numeric_limits::max() - step) { + auto column = ColumnUInt64::create(0); + return {Columns{std::move(column)}, 0}; + } + first_element += step; + } if (first_element - curr >= block_size) { auto column = ColumnUInt64::create(0); - return {Columns{std::move(column)}, filtered_block_size}; + return {Columns{std::move(column)}, 0}; } - if (first_element - curr < block_size) - filtered_block_size = (block_size - (first_element - curr) - 1) / inner_step + 1; + UInt64 filtered_block_size = (block_size - (first_element - curr) - 1) / step + 1; auto column = ColumnUInt64::create(filtered_block_size); ColumnUInt64::Container & vec = column->getData(); UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * end = &vec[filtered_block_size]; - iota_with_step(pos, static_cast(end - pos), first_element, inner_step); + iota_with_step(pos, static_cast(end - pos), first_element, step); - next += step; + next += chunk_step; progress(column->size(), column->byteSize()); @@ -76,9 +86,9 @@ protected: private: UInt64 block_size; UInt64 next; + UInt64 chunk_step; UInt64 step; - UInt64 inner_step; - UInt64 inner_remainder; + UInt64 remainder; }; struct RangeWithStep @@ -565,7 +575,8 @@ Pipe ReadFromSystemNumbersStep::makePipe() numbers_storage.offset + i * max_block_size, num_streams * max_block_size, numbers_storage.column_name, - numbers_storage.step); + numbers_storage.step, + numbers_storage.offset % numbers_storage.step); if (numbers_storage.limit && i == 0) { diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index 298721984b8..c698bae4393 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -10,6 +10,34 @@ namespace DB class Context; +/** Implements a table engine for the system table "numbers". + * The table contains the only column number UInt64. + * From this table, you can read all natural numbers, starting from 0 (to 2^64 - 1, and then again). + * + * You could also specify a limit (how many numbers to give). + * + * How to generate numbers? + * + * 1. First try a smart fashion: + * + * In this fashion we try to push filters and limit down to scanning. + * Firstly extract plain ranges(no overlapping and ordered) by filter expressions. + * + * For example: + * where (numbers > 1 and numbers < 3) or (numbers in (4, 6)) or (numbers > 7 and numbers < 9) + * + * We will get ranges + * (1, 3), [4, 4], [6, 6], (7, 9) + * + * Then split the ranges evenly to one or multi-streams. With this way we will get result without large scanning. + * + * 2. If fail to extract plain ranges, fall back to ordinary scanning. + * + * If multithreaded is specified, numbers will be generated in several streams + * (and result could be out of order). If both multithreaded and limit are specified, + * the table could give you not exactly 1..limit range, but some arbitrary 'limit' numbers. + */ + class StorageSystemNumbers final : public IStorage { public: From 696609e7d562d15cfc7a6ffa776785444a97c2e7 Mon Sep 17 00:00:00 2001 From: divanik Date: Sat, 10 Feb 2024 19:59:57 +0000 Subject: [PATCH 0024/1165] Kek --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 2488fa37643..dc6aebc69c1 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -14,6 +14,7 @@ #include #include +#include "Core/Types.h" #include "base/types.h" namespace DB @@ -51,13 +52,16 @@ protected: { UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. UInt64 first_element = (curr / step) * step; - if (first_element > std::numeric_limits::max() - remainder) { + if (first_element > std::numeric_limits::max() - remainder) + { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, 0}; } first_element += remainder; - if (first_element < curr) { - if (first_element > std::numeric_limits::max() - step) { + if (first_element < curr) + { + if (first_element > std::numeric_limits::max() - step) + { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, 0}; } @@ -101,6 +105,8 @@ using RangesWithStep = std::vector; std::optional stepped_range_from_range(const Range & r, UInt64 step, UInt64 remainder) { + // LOG_DEBUG(&Poco::Logger::get("Stepped from range"), + // "stepped from range"); if ((r.right.get() == 0) && (!r.right_included)) return std::nullopt; UInt64 begin = (r.left.get() / step) * step; @@ -126,7 +132,11 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st if (r.range.right.isPositiveInfinity()) return static_cast(std::numeric_limits::max() - r.range.left.get()) / r.step + r.range.left_included; - return static_cast(r.range.right.get() - r.range.left.get()) / r.step + 1; + UInt128 size = static_cast(r.range.right.get() - r.range.left.get()) / r.step; + if (r.range.right_included && (r.range.right.get() % r.step == 0)) { + ++size; + } + return size; }; [[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) @@ -173,6 +183,17 @@ public: , base_block_size(base_block_size_) , step(step_) { + // for (const auto& range_with_step : ranges_) { + // // LOG_DEBUG(&Poco::Logger::get("Ranges With Step"), + // // "Ranges: {} {} {} {} {}", + // // range_with_step.range.left.get(), + // // range_with_step.range.right.get(), + // // range_with_step.range.left_included, + // // range_with_step.range.right_included, + // // range_with_step.step); + // // LOG_DEBUG(&Poco::Logger::get("Ranges With Step"), + // // "Step: {}", step); + // } } String getName() const override { return "NumbersRange"; } @@ -241,6 +262,8 @@ protected: RangesPos start, end; auto block_size = findRanges(start, end, base_block_size); + // LOG_DEBUG(&Poco::Logger::get("Found range"), "Evth: {} {} {} {} {} {}", start.offset_in_ranges, static_cast(start.offset_in_range), end.offset_in_ranges, static_cast(end.offset_in_range), base_block_size, block_size); + if (!block_size) return {}; @@ -256,6 +279,11 @@ protected: while (block_size - provided != 0) { UInt64 need = block_size - provided; + // LOG_DEBUG(&Poco::Logger::get("Indices:"), + // "Indices: {} {}, provided: {}", + // ranges.size(), + // cursor.offset_in_ranges, + // provided); auto & range = ranges[cursor.offset_in_ranges]; UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges @@ -445,13 +473,15 @@ Pipe ReadFromSystemNumbersStep::makePipe() Pipe pipe; Ranges ranges; - // LOG_DEBUG(&Poco::Logger::get("parameters"), "Parameters: {} {} {}", numbers_storage.step, numbers_storage.limit.value(), numbers_storage.offset); + + // LOG_DEBUG(&Poco::Logger::get("parameters"), "Parameters: {} {} {} {}", numbers_storage.step, numbers_storage.offset, numbers_storage.limit.has_value(), numbers_storage.limit.has_value() ? numbers_storage.limit.value() : UInt64{0}); if (numbers_storage.limit.has_value() && (numbers_storage.limit.value() == 0)) { pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); return pipe; } + chassert(numbers_storage.step != UInt64{0}); /// Build rpn of query filters KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); @@ -575,7 +605,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() numbers_storage.offset + i * max_block_size, num_streams * max_block_size, numbers_storage.column_name, - numbers_storage.step, + numbers_storage.step, numbers_storage.offset % numbers_storage.step); if (numbers_storage.limit && i == 0) From 3ec9f3c4c89dec2f1971979d7d3ae406c1ecd938 Mon Sep 17 00:00:00 2001 From: divanik Date: Sat, 10 Feb 2024 20:06:52 +0000 Subject: [PATCH 0025/1165] Check foormattign --- src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index dc6aebc69c1..1e6b539ee2e 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -52,15 +52,15 @@ protected: { UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. UInt64 first_element = (curr / step) * step; - if (first_element > std::numeric_limits::max() - remainder) + if (first_element > std::numeric_limits::max() - remainder) { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, 0}; } first_element += remainder; - if (first_element < curr) + if (first_element < curr) { - if (first_element > std::numeric_limits::max() - step) + if (first_element > std::numeric_limits::max() - step) { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, 0}; @@ -133,9 +133,8 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st return static_cast(std::numeric_limits::max() - r.range.left.get()) / r.step + r.range.left_included; UInt128 size = static_cast(r.range.right.get() - r.range.left.get()) / r.step; - if (r.range.right_included && (r.range.right.get() % r.step == 0)) { + if (r.range.right_included && (r.range.right.get() % r.step == 0)) ++size; - } return size; }; From d0456980991c45935fd316ca7dc2bd61cf45e5b9 Mon Sep 17 00:00:00 2001 From: divanik Date: Sat, 10 Feb 2024 23:04:52 +0000 Subject: [PATCH 0026/1165] It seems to work --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 31 ++++++------------- ...ble_functions_must_be_documented.reference | 1 + 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 1e6b539ee2e..2217e426b02 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -97,7 +97,8 @@ private: struct RangeWithStep { - Range range; + UInt64 left; + UInt64 right; UInt64 step; }; @@ -124,18 +125,12 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st if ((begin >= r.right_included) && (begin - r.right_included >= r.right.get())) return std::nullopt; UInt64 right_edge_included = r.right.get() - (1 - r.right_included); - return std::optional{RangeWithStep{Range(begin, true, right_edge_included, true), step}}; + return std::optional{RangeWithStep{begin, right_edge_included, step}}; } [[maybe_unused]] UInt128 sizeOfRange(const RangeWithStep & r) { - if (r.range.right.isPositiveInfinity()) - return static_cast(std::numeric_limits::max() - r.range.left.get()) / r.step + r.range.left_included; - - UInt128 size = static_cast(r.range.right.get() - r.range.left.get()) / r.step; - if (r.range.right_included && (r.range.right.get() % r.step == 0)) - ++size; - return size; + return static_cast(r.right - r.left) / r.step + 1; }; [[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) @@ -252,10 +247,6 @@ protected: if (ranges.empty()) return {}; - auto first_value = [](const RangeWithStep & r) { return r.range.left.get() + (r.range.left_included ? 0 : 1); }; - - auto last_value = [](const RangeWithStep & r) { return r.range.right.get() - (r.range.right_included ? 0 : 1); }; - /// Find the data range. /// If data left is small, shrink block size. RangesPos start, end; @@ -287,7 +278,7 @@ protected: UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges ? end.offset_in_range - cursor.offset_in_range - : static_cast(last_value(range) - first_value(range)) / range.step + 1 - cursor.offset_in_range; + : static_cast(range.right - range.left) / range.step + 1 - cursor.offset_in_range; /// set value to block auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) @@ -312,7 +303,7 @@ protected: if (can_provide > need) { - UInt64 start_value = first_value(range) + cursor.offset_in_range * step; + UInt64 start_value = range.left + cursor.offset_in_range * step; /// end_value will never overflow iota_with_step(pos, static_cast(need), start_value, step); pos += need; @@ -323,7 +314,7 @@ protected: else if (can_provide == need) { /// to avoid UInt64 overflow - UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range * step; + UInt128 start_value = static_cast(range.left) + cursor.offset_in_range * step; UInt128 end_value = start_value + need * step; set_value(start_value, end_value); @@ -334,7 +325,7 @@ protected: else { /// to avoid UInt64 overflow - UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range * step; + UInt128 start_value = static_cast(range.left) + cursor.offset_in_range * step; UInt128 end_value = start_value + can_provide * step; set_value(start_value, end_value); @@ -400,9 +391,7 @@ namespace else { auto & range = ranges[i]; - UInt64 right = range.range.left.get() + static_cast(size); - range.range.right = Field(right); - range.range.right_included = !range.range.left_included; + range.right = range.left + static_cast(size) * range.step - 1; last_range_idx = i; break; } @@ -558,7 +547,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() const auto & limit_offset = limit_length_and_offset.second; /// If intersected ranges is limited or we can pushdown limit. - if (!intersected_ranges.rbegin()->range.right.isPositiveInfinity() || should_pushdown_limit) + if (should_pushdown_limit) { UInt128 total_size = sizeOfRanges(intersected_ranges); UInt128 query_limit = limit_length + limit_offset; diff --git a/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference b/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference index e4040a2d371..1e4f21a6722 100644 --- a/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference @@ -4,6 +4,7 @@ dictionary executable file generateRandom +generate_series input jdbc merge From 789d3c699c77d7a39f42281d9dc0c61010471242 Mon Sep 17 00:00:00 2001 From: divanik Date: Sat, 10 Feb 2024 23:49:58 +0000 Subject: [PATCH 0027/1165] Remove bug for mt --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 65 +++++++++---------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 2217e426b02..3656a6d31ee 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -547,42 +547,39 @@ Pipe ReadFromSystemNumbersStep::makePipe() const auto & limit_offset = limit_length_and_offset.second; /// If intersected ranges is limited or we can pushdown limit. - if (should_pushdown_limit) + UInt128 total_size = sizeOfRanges(intersected_ranges); + UInt128 query_limit = limit_length + limit_offset; + + /// limit total_size by query_limit + if (should_pushdown_limit && query_limit < total_size) { - UInt128 total_size = sizeOfRanges(intersected_ranges); - UInt128 query_limit = limit_length + limit_offset; - - /// limit total_size by query_limit - if (should_pushdown_limit && query_limit < total_size) - { - total_size = query_limit; - /// We should shrink intersected_ranges for case: - /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 - shrinkRanges(intersected_ranges, total_size); - } - - checkLimits(size_t(total_size)); - - if (total_size / max_block_size < num_streams) - num_streams = static_cast(total_size / max_block_size); - - if (num_streams == 0) - num_streams = 1; - - /// Ranges state, all streams will share the state. - auto ranges_state = std::make_shared(); - for (size_t i = 0; i < num_streams; ++i) - { - auto source = std::make_shared( - intersected_ranges, ranges_state, max_block_size, numbers_storage.step, numbers_storage.column_name); - - if (i == 0) - source->addTotalRowsApprox(total_size); - - pipe.addSource(std::move(source)); - } - return pipe; + total_size = query_limit; + /// We should shrink intersected_ranges for case: + /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 + shrinkRanges(intersected_ranges, total_size); } + + checkLimits(size_t(total_size)); + + if (total_size / max_block_size < num_streams) + num_streams = static_cast(total_size / max_block_size); + + if (num_streams == 0) + num_streams = 1; + + /// Ranges state, all streams will share the state. + auto ranges_state = std::make_shared(); + for (size_t i = 0; i < num_streams; ++i) + { + auto source = std::make_shared( + intersected_ranges, ranges_state, max_block_size, numbers_storage.step, numbers_storage.column_name); + + if (i == 0) + source->addTotalRowsApprox(total_size); + + pipe.addSource(std::move(source)); + } + return pipe; } /// Fall back to NumbersSource From 0f84f68da77663e2adcce800cceefff5ab019b58 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 13 Feb 2024 09:59:39 +0000 Subject: [PATCH 0028/1165] Simplified code --- src/Common/iota.cpp | 2 +- src/Common/iota.h | 2 +- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 65 +++++-------------- .../02970_generate_series.reference | 1 - .../0_stateless/02970_generate_series.sql | 1 - 5 files changed, 20 insertions(+), 51 deletions(-) diff --git a/src/Common/iota.cpp b/src/Common/iota.cpp index 532c4bde76d..86c9e04bb06 100644 --- a/src/Common/iota.cpp +++ b/src/Common/iota.cpp @@ -60,6 +60,6 @@ template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UIn template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); #if defined(OS_DARWIN) -extern template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); +template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); #endif } diff --git a/src/Common/iota.h b/src/Common/iota.h index f40cde9d5db..8fa18be9769 100644 --- a/src/Common/iota.h +++ b/src/Common/iota.h @@ -38,6 +38,6 @@ extern template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_val extern template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); extern template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); #if defined(OS_DARWIN) -extern template void iota(size_t * begin, size_t count, size_t first_value, size_t step); +extern template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); #endif } diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 3656a6d31ee..d69e2b6ca5a 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -15,6 +15,7 @@ #include #include "Core/Types.h" +#include "base/Decimal_fwd.h" #include "base/types.h" namespace DB @@ -31,13 +32,12 @@ namespace class NumbersSource : public ISource { public: - NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 chunk_step_, const std::string & column_name, UInt64 step_, UInt64 remainder_) + NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 chunk_step_, const std::string & column_name, UInt64 step_) : ISource(createHeader(column_name)) , block_size(block_size_) , next(offset_) , chunk_step(chunk_step_) , step(step_) - , remainder(remainder_) { } String getName() const override { return "Numbers"; } @@ -50,41 +50,19 @@ public: protected: Chunk generate() override { - UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. - UInt64 first_element = (curr / step) * step; - if (first_element > std::numeric_limits::max() - remainder) - { - auto column = ColumnUInt64::create(0); - return {Columns{std::move(column)}, 0}; - } - first_element += remainder; - if (first_element < curr) - { - if (first_element > std::numeric_limits::max() - step) - { - auto column = ColumnUInt64::create(0); - return {Columns{std::move(column)}, 0}; - } - first_element += step; - } - if (first_element - curr >= block_size) - { - auto column = ColumnUInt64::create(0); - return {Columns{std::move(column)}, 0}; - } - UInt64 filtered_block_size = (block_size - (first_element - curr) - 1) / step + 1; - - auto column = ColumnUInt64::create(filtered_block_size); + auto column = ColumnUInt64::create(block_size); ColumnUInt64::Container & vec = column->getData(); + + UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. UInt64 * pos = vec.data(); /// This also accelerates the code. - UInt64 * end = &vec[filtered_block_size]; - iota_with_step(pos, static_cast(end - pos), first_element, step); + UInt64 * end = &vec[block_size]; + iota_with_step(pos, static_cast(end - pos), curr, step); next += chunk_step; progress(column->size(), column->byteSize()); - return {Columns{std::move(column)}, filtered_block_size}; + return {Columns{std::move(column)}, block_size}; } private: @@ -92,14 +70,13 @@ private: UInt64 next; UInt64 chunk_step; UInt64 step; - UInt64 remainder; }; struct RangeWithStep { UInt64 left; - UInt64 right; UInt64 step; + UInt128 size; }; using RangesWithStep = std::vector; @@ -125,21 +102,16 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st if ((begin >= r.right_included) && (begin - r.right_included >= r.right.get())) return std::nullopt; UInt64 right_edge_included = r.right.get() - (1 - r.right_included); - return std::optional{RangeWithStep{begin, right_edge_included, step}}; + return std::optional{RangeWithStep{begin, step, static_cast(right_edge_included - begin) / step + 1}}; } -[[maybe_unused]] UInt128 sizeOfRange(const RangeWithStep & r) -{ - return static_cast(r.right - r.left) / r.step + 1; -}; - [[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) { UInt128 total_size{}; for (const RangeWithStep & r : rs) { /// total_size will never overflow - total_size += sizeOfRange(r); + total_size += r.size; } return total_size; }; @@ -211,7 +183,7 @@ protected: while (need != 0) { UInt128 can_provide = end.offset_in_ranges == ranges.size() ? static_cast(0) - : sizeOfRange(ranges[end.offset_in_ranges]) - end.offset_in_range; + : ranges[end.offset_in_ranges].size - end.offset_in_range; if (can_provide == 0) break; @@ -278,7 +250,7 @@ protected: UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges ? end.offset_in_range - cursor.offset_in_range - : static_cast(range.right - range.left) / range.step + 1 - cursor.offset_in_range; + : range.size - cursor.offset_in_range; /// set value to block auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) @@ -377,7 +349,7 @@ namespace size_t last_range_idx = 0; for (size_t i = 0; i < ranges.size(); i++) { - auto range_size = sizeOfRange(ranges[i]); + auto range_size = ranges[i].size; if (range_size < size) { size -= static_cast(range_size); @@ -391,7 +363,7 @@ namespace else { auto & range = ranges[i]; - range.right = range.left + static_cast(size) * range.step - 1; + range.size = static_cast(size); last_range_idx = i; break; } @@ -587,11 +559,10 @@ Pipe ReadFromSystemNumbersStep::makePipe() { auto source = std::make_shared( max_block_size, - numbers_storage.offset + i * max_block_size, - num_streams * max_block_size, + numbers_storage.offset + i * max_block_size * numbers_storage.step, + num_streams * max_block_size * numbers_storage.step, numbers_storage.column_name, - numbers_storage.step, - numbers_storage.offset % numbers_storage.step); + numbers_storage.step); if (numbers_storage.limit && i == 0) { diff --git a/tests/queries/0_stateless/02970_generate_series.reference b/tests/queries/0_stateless/02970_generate_series.reference index 9e6f1db911e..4e4f556a39b 100644 --- a/tests/queries/0_stateless/02970_generate_series.reference +++ b/tests/queries/0_stateless/02970_generate_series.reference @@ -4,7 +4,6 @@ 8 500000001 50000000 -100000001 0 10 13 diff --git a/tests/queries/0_stateless/02970_generate_series.sql b/tests/queries/0_stateless/02970_generate_series.sql index 045f584a622..a7f89e1bd3f 100644 --- a/tests/queries/0_stateless/02970_generate_series.sql +++ b/tests/queries/0_stateless/02970_generate_series.sql @@ -4,7 +4,6 @@ SELECT count() FROM generate_series(10, 20, 3); SELECT count() FROM generate_series(7, 77, 10); SELECT count() FROM generate_series(0, 1000000000, 2); SELECT count() FROM generate_series(0, 999999999, 20); -SELECT count() FROM generate_series(0, 1000000000, 2) WHERE generate_series % 5 == 0; SELECT * FROM generate_series(5, 4); SELECT * FROM generate_series(0, 0); From d12ecdc5f06689d6259e2ef082a916f8b2f1836f Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 13 Feb 2024 12:35:17 +0100 Subject: [PATCH 0029/1165] Asynchronous WriteBuffer for AzureBlobStorage --- src/Backups/BackupIO_AzureBlobStorage.cpp | 3 +- src/Core/Settings.h | 3 +- .../IO/WriteBufferFromAzureBlobStorage.cpp | 116 +++++++++++------- .../IO/WriteBufferFromAzureBlobStorage.h | 22 +++- .../AzureBlobStorage/AzureBlobStorageAuth.cpp | 3 +- .../AzureBlobStorage/AzureObjectStorage.cpp | 3 +- .../AzureBlobStorage/AzureObjectStorage.h | 5 +- src/IO/WriteBufferFromS3.cpp | 2 +- src/IO/WriteBufferFromS3.h | 5 +- src/IO/WriteBufferFromS3TaskTracker.cpp | 21 ++-- src/IO/WriteBufferFromS3TaskTracker.h | 8 +- 11 files changed, 124 insertions(+), 67 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 52ce20d5108..44a72f80456 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -278,7 +278,8 @@ std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const Strin settings->max_single_part_upload_size, settings->max_unexpected_write_error_retries, DBMS_DEFAULT_BUFFER_SIZE, - write_settings); + write_settings, + settings->max_inflight_parts_for_one_file); } void BackupWriterAzureBlobStorage::removeFile(const String & file_name) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 44badfefabb..53de245bdfc 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -80,7 +80,8 @@ class IColumn; M(UInt64, s3_max_upload_part_size, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to S3.", 0) \ M(UInt64, s3_upload_part_size_multiply_factor, 2, "Multiply s3_min_upload_part_size by this factor each time s3_multiply_parts_count_threshold parts were uploaded from a single write to S3.", 0) \ M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, 500, "Each time this number of parts was uploaded to S3, s3_min_upload_part_size is multiplied by s3_upload_part_size_multiply_factor.", 0) \ - M(UInt64, s3_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited. You ", 0) \ + M(UInt64, s3_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \ + M(UInt64, azure_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \ M(UInt64, s3_max_single_part_upload_size, 32*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \ M(UInt64, azure_max_single_part_upload_size, 100*1024*1024, "The maximum size of object to upload using singlepart upload to Azure blob storage.", 0) \ M(UInt64, azure_max_single_part_copy_size, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage.", 0) \ diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index 905114f50e9..cbe2367823d 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -18,13 +18,21 @@ namespace ProfileEvents namespace DB { +struct WriteBufferFromAzureBlobStorage::PartData +{ + Memory<> memory; + size_t data_size = 0; +}; + WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( std::shared_ptr blob_container_client_, const String & blob_path_, size_t max_single_part_upload_size_, size_t max_unexpected_write_error_retries_, size_t buf_size_, - const WriteSettings & write_settings_) + const WriteSettings & write_settings_, + size_t max_inflight_parts_for_one_file_, + ThreadPoolCallbackRunner schedule_) : WriteBufferFromFileBase(buf_size_, nullptr, 0) , log(getLogger("WriteBufferFromAzureBlobStorage")) , max_single_part_upload_size(max_single_part_upload_size_) @@ -32,7 +40,13 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( , blob_path(blob_path_) , write_settings(write_settings_) , blob_container_client(blob_container_client_) + , task_tracker( + std::make_unique( + std::move(schedule_), + max_inflight_parts_for_one_file_, + limitedLog)) { + allocateBuffer(); } @@ -79,60 +93,80 @@ void WriteBufferFromAzureBlobStorage::finalizeImpl() { execWithRetry([this](){ next(); }, max_unexpected_write_error_retries); - if (tmp_buffer_write_offset > 0) - uploadBlock(tmp_buffer->data(), tmp_buffer_write_offset); + task_tracker->waitAll(); auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); execWithRetry([&](){ block_blob_client.CommitBlockList(block_ids); }, max_unexpected_write_error_retries); - LOG_TRACE(log, "Committed {} blocks for blob `{}`", block_ids.size(), blob_path); -} - -void WriteBufferFromAzureBlobStorage::uploadBlock(const char * data, size_t size) -{ - auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); - const std::string & block_id = block_ids.emplace_back(getRandomASCIIString(64)); - - Azure::Core::IO::MemoryBodyStream memory_stream(reinterpret_cast(data), size); - execWithRetry([&](){ block_blob_client.StageBlock(block_id, memory_stream); }, max_unexpected_write_error_retries, size); - tmp_buffer_write_offset = 0; - - LOG_TRACE(log, "Staged block (id: {}) of size {} (blob path: {}).", block_id, size, blob_path); -} - -WriteBufferFromAzureBlobStorage::MemoryBufferPtr WriteBufferFromAzureBlobStorage::allocateBuffer() const -{ - return std::make_unique>(max_single_part_upload_size); + LOG_DEBUG(log, "Committed {} blocks for blob `{}`", block_ids.size(), blob_path); } void WriteBufferFromAzureBlobStorage::nextImpl() { - size_t size_to_upload = offset(); + task_tracker->waitIfAny(); - if (size_to_upload == 0) - return; + reallocateBuffer(); + detachBuffer(); - if (!tmp_buffer) - tmp_buffer = allocateBuffer(); - - size_t uploaded_size = 0; - while (uploaded_size != size_to_upload) + while (!detached_part_data.empty()) { - size_t memory_buffer_remaining_size = max_single_part_upload_size - tmp_buffer_write_offset; - if (memory_buffer_remaining_size == 0) - uploadBlock(tmp_buffer->data(), tmp_buffer->size()); - - size_t size = std::min(memory_buffer_remaining_size, size_to_upload - uploaded_size); - memcpy(tmp_buffer->data() + tmp_buffer_write_offset, working_buffer.begin() + uploaded_size, size); - uploaded_size += size; - tmp_buffer_write_offset += size; + writePart(std::move(detached_part_data.front())); + detached_part_data.pop_front(); } - if (tmp_buffer_write_offset == max_single_part_upload_size) - uploadBlock(tmp_buffer->data(), tmp_buffer->size()); + allocateBuffer(); +} - if (write_settings.remote_throttler) - write_settings.remote_throttler->add(size_to_upload, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); +void WriteBufferFromAzureBlobStorage::allocateBuffer() +{ + memory = Memory(max_single_part_upload_size); + WriteBuffer::set(memory.data(), memory.size()); +} + + +void WriteBufferFromAzureBlobStorage::reallocateBuffer() +{ + chassert(offset() == 0); + + if (available() > 0) + return; + + if (memory.size() == max_single_part_upload_size) + return; + + memory.resize(max_single_part_upload_size); + + WriteBuffer::set(memory.data(), memory.size()); + + chassert(offset() == 0); +} + +void WriteBufferFromAzureBlobStorage::detachBuffer() +{ + size_t data_size = size_t(position() - memory.data()); + auto buf = std::move(memory); + WriteBuffer::set(nullptr, 0); + detached_part_data.push_back({std::move(buf), data_size}); +} + +void WriteBufferFromAzureBlobStorage::writePart(WriteBufferFromAzureBlobStorage::PartData && data) +{ + if (data.data_size == 0) + return; + + auto upload_worker = [&] () + { + auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); + const std::string & block_id = block_ids.emplace_back(getRandomASCIIString(64)); + + Azure::Core::IO::MemoryBodyStream memory_stream(reinterpret_cast(data.memory.data()), data.data_size); + execWithRetry([&](){ block_blob_client.StageBlock(block_id, memory_stream); }, max_unexpected_write_error_retries, data.data_size); + + if (write_settings.remote_throttler) + write_settings.remote_throttler->add(data.data_size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); + }; + + task_tracker->add(std::move(upload_worker)); } } diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index f105b35c121..2d11014fa2a 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace Poco @@ -21,6 +22,8 @@ class Logger; namespace DB { +class TaskTracker; + class WriteBufferFromAzureBlobStorage : public WriteBufferFromFileBase { public: @@ -32,7 +35,9 @@ public: size_t max_single_part_upload_size_, size_t max_unexpected_write_error_retries_, size_t buf_size_, - const WriteSettings & write_settings_); + const WriteSettings & write_settings_, + size_t max_inflight_parts_for_one_file_, + ThreadPoolCallbackRunner schedule_ = {}); ~WriteBufferFromAzureBlobStorage() override; @@ -42,11 +47,21 @@ public: void sync() override { next(); } private: + struct PartData; + + void writePart(WriteBufferFromAzureBlobStorage::PartData && data); + void detachBuffer(); + void allocateBuffer(); + void allocateFirstBuffer(); + void reallocateFirstBuffer(); + void reallocateBuffer(); + void finalizeImpl() override; void execWithRetry(std::function func, size_t num_tries, size_t cost = 0); void uploadBlock(const char * data, size_t size); LoggerPtr log; + LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); const size_t max_single_part_upload_size; const size_t max_unexpected_write_error_retries; @@ -61,6 +76,11 @@ private: size_t tmp_buffer_write_offset = 0; MemoryBufferPtr allocateBuffer() const; + + bool first_buffer=true; + + std::unique_ptr task_tracker; + std::deque detached_part_data; }; } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index 72c4abee5c9..f99586b2d1a 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -169,7 +169,8 @@ std::unique_ptr getAzureBlobStorageSettings(const Po config.getUInt64(config_prefix + ".max_upload_part_size", 5ULL * 1024 * 1024 * 1024), config.getUInt64(config_prefix + ".max_single_part_copy_size", context->getSettings().azure_max_single_part_copy_size), config.getBool(config_prefix + ".use_native_copy", false), - config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", context->getSettings().azure_max_unexpected_write_error_retries) + config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", context->getSettings().azure_max_unexpected_write_error_retries), + config.getUInt64(config_prefix + ".max_inflight_parts_for_one_file", context->getSettings().azure_max_inflight_parts_for_one_file) ); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index 74389aedb64..844789ea5b5 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -268,7 +268,8 @@ std::unique_ptr AzureObjectStorage::writeObject( /// NO settings.get()->max_single_part_upload_size, settings.get()->max_unexpected_write_error_retries, buf_size, - patchSettings(write_settings)); + patchSettings(write_settings), + settings.get()->max_inflight_parts_for_one_file); } /// Remove file. Throws exception if file doesn't exists or it's a directory. diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index f16c35fb52c..1b473a01304 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -27,7 +27,8 @@ struct AzureObjectStorageSettings size_t max_upload_part_size_, size_t max_single_part_copy_size_, bool use_native_copy_, - size_t max_unexpected_write_error_retries_) + size_t max_unexpected_write_error_retries_, + size_t max_inflight_parts_for_one_file_) : max_single_part_upload_size(max_single_part_upload_size_) , min_bytes_for_seek(min_bytes_for_seek_) , max_single_read_retries(max_single_read_retries_) @@ -37,6 +38,7 @@ struct AzureObjectStorageSettings , max_single_part_copy_size(max_single_part_copy_size_) , use_native_copy(use_native_copy_) , max_unexpected_write_error_retries (max_unexpected_write_error_retries_) + , max_inflight_parts_for_one_file (max_inflight_parts_for_one_file_) { } @@ -52,6 +54,7 @@ struct AzureObjectStorageSettings size_t max_single_part_copy_size = 256 * 1024 * 1024; bool use_native_copy = false; size_t max_unexpected_write_error_retries = 4; + size_t max_inflight_parts_for_one_file = 20; }; using AzureClient = Azure::Storage::Blobs::BlobContainerClient; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 5bb01050591..6fc0a35672f 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -95,7 +95,7 @@ WriteBufferFromS3::WriteBufferFromS3( , object_metadata(std::move(object_metadata_)) , buffer_allocation_policy(ChooseBufferPolicy(upload_settings)) , task_tracker( - std::make_unique( + std::make_unique( std::move(schedule_), upload_settings.max_inflight_parts_for_one_file, limitedLog)) diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 230f39b074e..f3637122ee4 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,8 @@ namespace DB * Data is divided on chunks with size greater than 'minimum_upload_part_size'. Last chunk can be less than this threshold. * Each chunk is written as a part to S3. */ +class TaskTracker; + class WriteBufferFromS3 final : public WriteBufferFromFileBase { public: @@ -118,7 +121,7 @@ private: size_t total_size = 0; size_t hidden_size = 0; - class TaskTracker; +// class TaskTracker; std::unique_ptr task_tracker; BlobStorageLogWriterPtr blob_log; diff --git a/src/IO/WriteBufferFromS3TaskTracker.cpp b/src/IO/WriteBufferFromS3TaskTracker.cpp index bce122dd6c8..e62de261fc2 100644 --- a/src/IO/WriteBufferFromS3TaskTracker.cpp +++ b/src/IO/WriteBufferFromS3TaskTracker.cpp @@ -1,7 +1,5 @@ #include "config.h" -#if USE_AWS_S3 - #include namespace ProfileEvents @@ -12,19 +10,19 @@ namespace ProfileEvents namespace DB { -WriteBufferFromS3::TaskTracker::TaskTracker(ThreadPoolCallbackRunner scheduler_, size_t max_tasks_inflight_, LogSeriesLimiterPtr limitedLog_) +TaskTracker::TaskTracker(ThreadPoolCallbackRunner scheduler_, size_t max_tasks_inflight_, LogSeriesLimiterPtr limitedLog_) : is_async(bool(scheduler_)) , scheduler(scheduler_ ? std::move(scheduler_) : syncRunner()) , max_tasks_inflight(max_tasks_inflight_) , limitedLog(limitedLog_) {} -WriteBufferFromS3::TaskTracker::~TaskTracker() +TaskTracker::~TaskTracker() { safeWaitAll(); } -ThreadPoolCallbackRunner WriteBufferFromS3::TaskTracker::syncRunner() +ThreadPoolCallbackRunner TaskTracker::syncRunner() { return [](Callback && callback, int64_t) mutable -> std::future { @@ -35,7 +33,7 @@ ThreadPoolCallbackRunner WriteBufferFromS3::TaskTracker::syncRunner() }; } -void WriteBufferFromS3::TaskTracker::waitAll() +void TaskTracker::waitAll() { /// Exceptions are propagated for (auto & future : futures) @@ -48,7 +46,7 @@ void WriteBufferFromS3::TaskTracker::waitAll() finished_futures.clear(); } -void WriteBufferFromS3::TaskTracker::safeWaitAll() +void TaskTracker::safeWaitAll() { for (auto & future : futures) { @@ -71,7 +69,7 @@ void WriteBufferFromS3::TaskTracker::safeWaitAll() finished_futures.clear(); } -void WriteBufferFromS3::TaskTracker::waitIfAny() +void TaskTracker::waitIfAny() { if (futures.empty()) return; @@ -99,7 +97,7 @@ void WriteBufferFromS3::TaskTracker::waitIfAny() ProfileEvents::increment(ProfileEvents::WriteBufferFromS3WaitInflightLimitMicroseconds, watch.elapsedMicroseconds()); } -void WriteBufferFromS3::TaskTracker::add(Callback && func) +void TaskTracker::add(Callback && func) { /// All this fuzz is about 2 things. This is the most critical place of TaskTracker. /// The first is not to fail insertion in the list `futures`. @@ -134,7 +132,7 @@ void WriteBufferFromS3::TaskTracker::add(Callback && func) waitTilInflightShrink(); } -void WriteBufferFromS3::TaskTracker::waitTilInflightShrink() +void TaskTracker::waitTilInflightShrink() { if (!max_tasks_inflight) return; @@ -166,11 +164,10 @@ void WriteBufferFromS3::TaskTracker::waitTilInflightShrink() ProfileEvents::increment(ProfileEvents::WriteBufferFromS3WaitInflightLimitMicroseconds, watch.elapsedMicroseconds()); } -bool WriteBufferFromS3::TaskTracker::isAsync() const +bool TaskTracker::isAsync() const { return is_async; } } -#endif diff --git a/src/IO/WriteBufferFromS3TaskTracker.h b/src/IO/WriteBufferFromS3TaskTracker.h index 815e041ae52..134abbbc4c1 100644 --- a/src/IO/WriteBufferFromS3TaskTracker.h +++ b/src/IO/WriteBufferFromS3TaskTracker.h @@ -1,9 +1,7 @@ #pragma once #include "config.h" - -#if USE_AWS_S3 - +#include #include "WriteBufferFromS3.h" #include @@ -22,7 +20,7 @@ namespace DB /// Basic exception safety is provided. If exception occurred the object has to be destroyed. /// No thread safety is provided. Use this object with no concurrency. -class WriteBufferFromS3::TaskTracker +class TaskTracker { public: using Callback = std::function; @@ -68,5 +66,3 @@ private: }; } - -#endif From 26fd3d0d852986b6bbaf595087cb0d06bdff9f93 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 14 Feb 2024 16:13:53 +0100 Subject: [PATCH 0030/1165] Removed offset check --- src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index cbe2367823d..d700090303a 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -126,8 +126,6 @@ void WriteBufferFromAzureBlobStorage::allocateBuffer() void WriteBufferFromAzureBlobStorage::reallocateBuffer() { - chassert(offset() == 0); - if (available() > 0) return; From 7bf42fd86e9599357282f947312c98d2bec1047f Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 16 Feb 2024 11:16:14 +0100 Subject: [PATCH 0031/1165] Fix upgrade check --- src/Core/SettingsChangesHistory.h | 3 ++- src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index c453dd837eb..b6d07d7057a 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -93,7 +93,8 @@ static std::map sett {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"}, {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}, - {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."}}}, + {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."}, + {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."}}}, {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"}, diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index d700090303a..74a8949b235 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -98,7 +98,7 @@ void WriteBufferFromAzureBlobStorage::finalizeImpl() auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); execWithRetry([&](){ block_blob_client.CommitBlockList(block_ids); }, max_unexpected_write_error_retries); - LOG_DEBUG(log, "Committed {} blocks for blob `{}`", block_ids.size(), blob_path); + LOG_TRACE(log, "Committed {} blocks for blob `{}`", block_ids.size(), blob_path); } void WriteBufferFromAzureBlobStorage::nextImpl() From 1549725eddb6db299ba0297de21a51411607d2a3 Mon Sep 17 00:00:00 2001 From: unashi Date: Sun, 18 Feb 2024 19:26:12 +0800 Subject: [PATCH 0032/1165] [feature]: allow to attach parts from a different disk --- src/Storages/MergeTree/MergeTreeData.cpp | 13 ++ src/Storages/MergeTree/MergeTreeData.h | 9 + .../MergeTree/MergeTreeDataPartCloner.cpp | 70 ++++++- src/Storages/StorageMergeTree.cpp | 45 +++-- src/Storages/StorageReplicatedMergeTree.cpp | 42 ++-- .../__init__.py | 0 .../configs/remote_servers.xml | 17 ++ .../test_attach_partition_using_copy/test.py | 183 ++++++++++++++++++ 8 files changed, 353 insertions(+), 26 deletions(-) create mode 100644 tests/integration/test_attach_partition_using_copy/__init__.py create mode 100644 tests/integration/test_attach_partition_using_copy/configs/remote_servers.xml create mode 100644 tests/integration/test_attach_partition_using_copy/test.py diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 3ca746a7197..56710b157de 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7085,6 +7085,19 @@ std::pair MergeTreeData::cloneAn this, src_part, metadata_snapshot, dst_part_info, tmp_part_prefix, require_part_metadata, params, read_settings, write_settings); } +std::pair MergeTreeData::cloneAndLoadDataPartOnOtherDisk( + const MergeTreeData::DataPartPtr & src_part, + const String & tmp_part_prefix, + const MergeTreePartInfo & dst_part_info, + const StorageMetadataPtr & metadata_snapshot, + const IDataPartStorage::ClonePartParams & params, + const ReadSettings & read_settings, + const WriteSettings & write_settings) +{ + return MergeTreeDataPartCloner::clone( + this, src_part, metadata_snapshot, dst_part_info, tmp_part_prefix, require_part_metadata, params, read_settings, write_settings); +} + std::pair MergeTreeData::cloneAndLoadPartOnSameDiskWithDifferentPartitionKey( const MergeTreeData::DataPartPtr & src_part, const MergeTreePartition & new_partition, diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index dfdc22baa8f..a24362f68fc 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -866,6 +866,15 @@ public: ContextPtr local_context, Int64 min_block, Int64 max_block); + + std::pair cloneAndLoadDataPartOnOtherDisk( + const MergeTreeData::DataPartPtr & src_part, + const String & tmp_part_prefix, + const MergeTreePartInfo & dst_part_info, + const StorageMetadataPtr & metadata_snapshot, + const IDataPartStorage::ClonePartParams & params, + const ReadSettings & read_settings, + const WriteSettings & write_settings); static std::pair createPartitionAndMinMaxIndexFromSourcePart( const MergeTreeData::DataPartPtr & src_part, diff --git a/src/Storages/MergeTree/MergeTreeDataPartCloner.cpp b/src/Storages/MergeTree/MergeTreeDataPartCloner.cpp index 04019d2c665..69b7abacc93 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCloner.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCloner.cpp @@ -142,6 +142,30 @@ std::shared_ptr hardlinkAllFiles( params); } +std::shared_ptr cloneAllFiles( + MergeTreeData * merge_tree_data, + const DB::ReadSettings & read_settings, + const DB::WriteSettings & write_settings, + const DataPartStoragePtr & storage, + const String & path) +{ + for (const DiskPtr & disk : merge_tree_data->getStoragePolicy()->getDisks()) + { + try{ + return storage->clonePart( + merge_tree_data->getRelativeDataPath(), + path, + disk, + read_settings, + write_settings,{},{}); + }catch(...) { + LOG_TRACE(&Poco::Logger::get("MergeTreeDataPartCloner"), "Clone part on disk {} fail", disk->getName()); + } + } + LOG_FATAL(&Poco::Logger::get("MergeTreeDataPartCloner"), "Clone part on disks all fail"); + throw; +} + std::pair cloneSourcePart( MergeTreeData * merge_tree_data, const MergeTreeData::DataPartPtr & src_part, @@ -165,8 +189,18 @@ std::pair cloneSourcePart( auto src_part_storage = flushPartStorageToDiskIfInMemory( merge_tree_data, src_part, metadata_snapshot, tmp_part_prefix, tmp_dst_part_name, src_flushed_tmp_dir_lock, src_flushed_tmp_part); - - auto dst_part_storage = hardlinkAllFiles(merge_tree_data, read_settings, write_settings, src_part_storage, tmp_dst_part_name, params); + std::shared_ptr dst_part_storage {}; + if (params.copy_instead_of_hardlink) { + dst_part_storage = cloneAllFiles(merge_tree_data, read_settings, write_settings, src_part_storage, tmp_dst_part_name); + } else { + try{ + dst_part_storage = hardlinkAllFiles(merge_tree_data, read_settings, write_settings, src_part_storage, tmp_dst_part_name, params); + } catch(...){ + // Hard link fail. Try copy. + LOG_WARNING(&Poco::Logger::get("MergeTreeDataPartCloner"), "Hard link fail, try tp copy directly. to:{}, path:{}", merge_tree_data->getRelativeDataPath(),tmp_dst_part_name); + dst_part_storage = cloneAllFiles(merge_tree_data, read_settings, write_settings, src_part_storage, tmp_dst_part_name); + } + } if (params.metadata_version_to_write.has_value()) { @@ -275,6 +309,25 @@ std::pair cloneAndHand return std::make_pair(destination_part, std::move(temporary_directory_lock)); } + +std::pair cloneInsteadOfHardlinksAndProjections( + MergeTreeData * merge_tree_data, + const DataPartPtr & src_part, + const StorageMetadataPtr & metadata_snapshot, + const MergeTreePartInfo & dst_part_info, + const String & tmp_part_prefix, + const ReadSettings & read_settings, + const WriteSettings & write_settings, + const IDataPartStorage::ClonePartParams & params) +{ + chassert(!merge_tree_data->isStaticStorage()); + + auto [destination_part, temporary_directory_lock] = cloneSourcePart( + merge_tree_data, src_part, metadata_snapshot, dst_part_info, tmp_part_prefix, read_settings, write_settings, params); + + return std::make_pair(destination_part, std::move(temporary_directory_lock)); +} + } std::pair MergeTreeDataPartCloner::clone( @@ -288,10 +341,19 @@ std::pair MergeTreeDat const ReadSettings & read_settings, const WriteSettings & write_settings) { - auto [destination_part, temporary_directory_lock] = cloneAndHandleHardlinksAndProjections( + if (params.copy_instead_of_hardlink) + { + auto [destination_part, temporary_directory_lock] = cloneInsteadOfHardlinksAndProjections( merge_tree_data, src_part, metadata_snapshot, dst_part_info, tmp_part_prefix, read_settings, write_settings, params); + return std::make_pair(finalizePart(destination_part, params, require_part_metadata), std::move(temporary_directory_lock)); + } + else + { + auto [destination_part, temporary_directory_lock] = cloneAndHandleHardlinksAndProjections( + merge_tree_data, src_part, metadata_snapshot, dst_part_info, tmp_part_prefix, read_settings, write_settings, params); + return std::make_pair(finalizePart(destination_part, params, require_part_metadata), std::move(temporary_directory_lock)); - return std::make_pair(finalizePart(destination_part, params, require_part_metadata), std::move(temporary_directory_lock)); + } } std::pair MergeTreeDataPartCloner::cloneWithDistinctPartitionExpression( diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 0f75c726bce..0f95fef9c6e 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2118,17 +2118,40 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con else { MergeTreePartInfo dst_part_info(partition_id, index, index, src_part->info.level); - - auto [dst_part, part_lock] = cloneAndLoadDataPartOnSameDisk( - src_part, - TMP_PREFIX, - dst_part_info, - my_metadata_snapshot, - clone_params, - local_context->getReadSettings(), - local_context->getWriteSettings()); - dst_parts.emplace_back(std::move(dst_part)); - dst_parts_locks.emplace_back(std::move(part_lock)); + LOG_TRACE(log, "Partition exps are the same:part id: {}; number of disks:{}",dst_part_info.partition_id, this->getStoragePolicy()->getDisks().size()); + bool on_same_disk = false; + for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) + { + if (disk->getName() == src_part->getDataPartStorage().getDiskName()) + on_same_disk = true; + } + if (on_same_disk) + { + auto [dst_part, part_lock] = cloneAndLoadDataPartOnSameDisk( + src_part, + TMP_PREFIX, + dst_part_info, + my_metadata_snapshot, + clone_params, + local_context->getReadSettings(), + local_context->getWriteSettings()); + dst_parts.emplace_back(std::move(dst_part)); + dst_parts_locks.emplace_back(std::move(part_lock)); + } + else + { + clone_params.copy_instead_of_hardlink = true; + auto [dst_part, part_lock] = cloneAndLoadDataPartOnOtherDisk( + src_part, + TMP_PREFIX, + dst_part_info, + my_metadata_snapshot, + clone_params, + local_context->getReadSettings(), + local_context->getWriteSettings()); + dst_parts.emplace_back(std::move(dst_part)); + dst_parts_locks.emplace_back(std::move(part_lock)); + } } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6bd57cc4d6d..ba0d27fe612 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8015,17 +8015,37 @@ void StorageReplicatedMergeTree::replacePartitionFrom( { MergeTreePartInfo dst_part_info(partition_id, index, index, src_part->info.level); - auto [dst_part, part_lock] = cloneAndLoadDataPartOnSameDisk( - src_part, - TMP_PREFIX, - dst_part_info, - metadata_snapshot, - clone_params, - query_context->getReadSettings(), - query_context->getWriteSettings()); - - dst_parts.emplace_back(dst_part); - dst_parts_locks.emplace_back(std::move(part_lock)); + bool on_same_disk = false; + for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) + if (disk->getName() == src_part->getDataPartStorage().getDiskName()) + on_same_disk = true; + if (on_same_disk) + { + auto [dst_part, part_lock] = cloneAndLoadDataPartOnSameDisk( + src_part, + TMP_PREFIX, + dst_part_info, + metadata_snapshot, + clone_params, + query_context->getReadSettings(), + query_context->getWriteSettings()); + dst_parts.emplace_back(dst_part); + dst_parts_locks.emplace_back(std::move(part_lock)); + } + else + { + clone_params.copy_instead_of_hardlink = true; + auto [dst_part, part_lock] = cloneAndLoadDataPartOnOtherDisk( + src_part, + TMP_PREFIX, + dst_part_info, + metadata_snapshot, + clone_params, + query_context->getReadSettings(), + query_context->getWriteSettings()); + dst_parts.emplace_back(dst_part); + dst_parts_locks.emplace_back(std::move(part_lock)); + } } src_parts.emplace_back(src_part); diff --git a/tests/integration/test_attach_partition_using_copy/__init__.py b/tests/integration/test_attach_partition_using_copy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_attach_partition_using_copy/configs/remote_servers.xml b/tests/integration/test_attach_partition_using_copy/configs/remote_servers.xml new file mode 100644 index 00000000000..b40730e9f7d --- /dev/null +++ b/tests/integration/test_attach_partition_using_copy/configs/remote_servers.xml @@ -0,0 +1,17 @@ + + + + + true + + replica1 + 9000 + + + replica2 + 9000 + + + + + diff --git a/tests/integration/test_attach_partition_using_copy/test.py b/tests/integration/test_attach_partition_using_copy/test.py new file mode 100644 index 00000000000..effb5708cf3 --- /dev/null +++ b/tests/integration/test_attach_partition_using_copy/test.py @@ -0,0 +1,183 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) + +replica1 = cluster.add_instance( + "replica1", with_zookeeper=True, main_configs=["configs/remote_servers.xml"] +) +replica2 = cluster.add_instance( + "replica2", with_zookeeper=True, main_configs=["configs/remote_servers.xml"] +) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + except Exception as ex: + print(ex) + finally: + cluster.shutdown() + + +def cleanup(nodes): + for node in nodes: + node.query("DROP TABLE IF EXISTS source SYNC") + node.query("DROP TABLE IF EXISTS destination SYNC") + + +def create_source_table(node, table_name, replicated): + replica = node.name + engine = ( + f"ReplicatedMergeTree('/clickhouse/tables/1/{table_name}', '{replica}')" + if replicated + else "MergeTree()" + ) + node.query_with_retry( + """ + ATTACH TABLE {table_name} UUID 'cf712b4f-2ca8-435c-ac23-c4393efe52f7' + ( + price UInt32, + date Date, + postcode1 LowCardinality(String), + postcode2 LowCardinality(String), + type Enum8('other' = 0, 'terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4), + is_new UInt8, + duration Enum8('unknown' = 0, 'freehold' = 1, 'leasehold' = 2), + addr1 String, + addr2 String, + street LowCardinality(String), + locality LowCardinality(String), + town LowCardinality(String), + district LowCardinality(String), + county LowCardinality(String) + ) + ENGINE = {engine} + ORDER BY (postcode1, postcode2, addr1, addr2) + SETTINGS disk = disk(type = web, endpoint = 'https://raw.githubusercontent.com/ClickHouse/web-tables-demo/main/web/') + """.format( + table_name=table_name, + engine=engine + ) + ) + + + +def create_destination_table(node, table_name, replicated): + replica = node.name + engine = ( + f"ReplicatedMergeTree('/clickhouse/tables/1/{table_name}', '{replica}')" + if replicated + else "MergeTree()" + ) + node.query_with_retry( + """ + CREATE TABLE {table_name} + ( + price UInt32, + date Date, + postcode1 LowCardinality(String), + postcode2 LowCardinality(String), + type Enum8('other' = 0, 'terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4), + is_new UInt8, + duration Enum8('unknown' = 0, 'freehold' = 1, 'leasehold' = 2), + addr1 String, + addr2 String, + street LowCardinality(String), + locality LowCardinality(String), + town LowCardinality(String), + district LowCardinality(String), + county LowCardinality(String) + ) + ENGINE = {engine} + ORDER BY (postcode1, postcode2, addr1, addr2) + """.format( + table_name=table_name, + engine=engine + ) + ) + +def test_both_mergtree(start_cluster): + create_source_table(replica1, "source", False) + create_destination_table(replica1, "destination", False) + + replica1.query( + f"ALTER TABLE destination ATTACH PARTITION tuple() FROM source" + ) + + assert_eq_with_retry( + replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC", + replica1.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC"), + ) + + assert_eq_with_retry( + replica1, f"SELECT town from destination LIMIT 1", + "SCARBOROUGH" + ) + + cleanup([replica1]) + +def test_all_replicated(start_cluster): + create_source_table(replica1, "source", True) + create_destination_table(replica1, "destination", True) + create_destination_table(replica2, "destination", True) + + replica1.query("SYSTEM SYNC REPLICA destination") + replica1.query( + f"ALTER TABLE destination ATTACH PARTITION tuple() FROM source" + ) + + assert_eq_with_retry( + replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC", + replica1.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC"), + ) + assert_eq_with_retry( + replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC", + replica2.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC"), + ) + + assert_eq_with_retry( + replica1, f"SELECT town from destination LIMIT 1", + "SCARBOROUGH" + ) + + assert_eq_with_retry( + replica2, f"SELECT town from destination LIMIT 1", + "SCARBOROUGH" + ) + + cleanup([replica1, replica2]) + +def test_only_destination_replicated(start_cluster): + create_source_table(replica1, "source", False) + create_destination_table(replica1, "destination", True) + create_destination_table(replica2, "destination", True) + + replica1.query("SYSTEM SYNC REPLICA destination") + replica1.query( + f"ALTER TABLE destination ATTACH PARTITION tuple() FROM source" + ) + + assert_eq_with_retry( + replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC", + replica1.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC"), + ) + assert_eq_with_retry( + replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC", + replica2.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC"), + ) + + assert_eq_with_retry( + replica1, f"SELECT town from destination LIMIT 1", + "SCARBOROUGH" + ) + + assert_eq_with_retry( + replica2, f"SELECT town from destination LIMIT 1", + "SCARBOROUGH" + ) + + cleanup([replica1, replica2]) From 80fe3f78d99caeaed733548ca65b6bd466730d51 Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 20 Feb 2024 11:12:09 +0800 Subject: [PATCH 0033/1165] [fix] black the python script --- .../test_attach_partition_using_copy/test.py | 92 ++++++++++--------- 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/tests/integration/test_attach_partition_using_copy/test.py b/tests/integration/test_attach_partition_using_copy/test.py index effb5708cf3..df5378742ae 100644 --- a/tests/integration/test_attach_partition_using_copy/test.py +++ b/tests/integration/test_attach_partition_using_copy/test.py @@ -59,13 +59,11 @@ def create_source_table(node, table_name, replicated): ORDER BY (postcode1, postcode2, addr1, addr2) SETTINGS disk = disk(type = web, endpoint = 'https://raw.githubusercontent.com/ClickHouse/web-tables-demo/main/web/') """.format( - table_name=table_name, - engine=engine + table_name=table_name, engine=engine ) ) - def create_destination_table(node, table_name, replicated): replica = node.name engine = ( @@ -95,89 +93,95 @@ def create_destination_table(node, table_name, replicated): ENGINE = {engine} ORDER BY (postcode1, postcode2, addr1, addr2) """.format( - table_name=table_name, - engine=engine + table_name=table_name, engine=engine ) ) + def test_both_mergtree(start_cluster): create_source_table(replica1, "source", False) create_destination_table(replica1, "destination", False) - replica1.query( - f"ALTER TABLE destination ATTACH PARTITION tuple() FROM source" - ) - + replica1.query(f"ALTER TABLE destination ATTACH PARTITION tuple() FROM source") + assert_eq_with_retry( - replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC", - replica1.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC"), + replica1, + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC", + replica1.query( + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC" + ), ) - + assert_eq_with_retry( - replica1, f"SELECT town from destination LIMIT 1", - "SCARBOROUGH" + replica1, f"SELECT town from destination LIMIT 1", "SCARBOROUGH" ) cleanup([replica1]) + def test_all_replicated(start_cluster): create_source_table(replica1, "source", True) create_destination_table(replica1, "destination", True) create_destination_table(replica2, "destination", True) replica1.query("SYSTEM SYNC REPLICA destination") - replica1.query( - f"ALTER TABLE destination ATTACH PARTITION tuple() FROM source" + replica1.query(f"ALTER TABLE destination ATTACH PARTITION tuple() FROM source") + + assert_eq_with_retry( + replica1, + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC", + replica1.query( + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC" + ), + ) + assert_eq_with_retry( + replica1, + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC", + replica2.query( + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC" + ), ) assert_eq_with_retry( - replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC", - replica1.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC"), - ) - assert_eq_with_retry( - replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC", - replica2.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC"), + replica1, f"SELECT town from destination LIMIT 1", "SCARBOROUGH" ) assert_eq_with_retry( - replica1, f"SELECT town from destination LIMIT 1", - "SCARBOROUGH" - ) - - assert_eq_with_retry( - replica2, f"SELECT town from destination LIMIT 1", - "SCARBOROUGH" + replica2, f"SELECT town from destination LIMIT 1", "SCARBOROUGH" ) cleanup([replica1, replica2]) + def test_only_destination_replicated(start_cluster): create_source_table(replica1, "source", False) create_destination_table(replica1, "destination", True) create_destination_table(replica2, "destination", True) replica1.query("SYSTEM SYNC REPLICA destination") - replica1.query( - f"ALTER TABLE destination ATTACH PARTITION tuple() FROM source" + replica1.query(f"ALTER TABLE destination ATTACH PARTITION tuple() FROM source") + + assert_eq_with_retry( + replica1, + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC", + replica1.query( + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC" + ), + ) + assert_eq_with_retry( + replica1, + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC", + replica2.query( + f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC" + ), ) assert_eq_with_retry( - replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC", - replica1.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC"), - ) - assert_eq_with_retry( - replica1, f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM source GROUP BY year ORDER BY year ASC", - replica2.query(f"SELECT toYear(date) AS year,round(avg(price)) AS price,bar(price, 0, 1000000, 80) FROM destination GROUP BY year ORDER BY year ASC"), + replica1, f"SELECT town from destination LIMIT 1", "SCARBOROUGH" ) assert_eq_with_retry( - replica1, f"SELECT town from destination LIMIT 1", - "SCARBOROUGH" - ) - - assert_eq_with_retry( - replica2, f"SELECT town from destination LIMIT 1", - "SCARBOROUGH" + replica2, f"SELECT town from destination LIMIT 1", "SCARBOROUGH" ) cleanup([replica1, replica2]) From 8de4a9dbfd32b7e82764a5c8efff3916b5c7ccda Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 20 Feb 2024 11:42:40 +0800 Subject: [PATCH 0034/1165] [fix] delete trailing whitespaces --- src/Storages/MergeTree/MergeTreeData.h | 2 +- src/Storages/StorageMergeTree.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 5f387385d38..081087acbaa 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -853,7 +853,7 @@ public: const IDataPartStorage::ClonePartParams & params, const ReadSettings & read_settings, const WriteSettings & write_settings); - + std::pair cloneAndLoadDataPartOnOtherDisk( const MergeTreeData::DataPartPtr & src_part, const String & tmp_part_prefix, diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index a2713775e65..47684925182 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2081,7 +2081,7 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con /// This will generate unique name in scope of current server process. Int64 temp_index = insert_increment.get(); MergeTreePartInfo dst_part_info(partition_id, temp_index, temp_index, src_part->info.level); - + IDataPartStorage::ClonePartParams clone_params{.txn = local_context->getCurrentTransaction()}; LOG_TRACE(log, "Partition exps are the same:part id: {}; number of disks:{}",dst_part_info.partition_id, this->getStoragePolicy()->getDisks().size()); bool on_same_disk = false; From 6437877a712bfaf4a36c180b332a0d6a37981af1 Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 20 Feb 2024 20:31:59 +0800 Subject: [PATCH 0035/1165] [fix] add changelog; change some feature logic --- CHANGELOG.md | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 10 +++++++--- src/Storages/StorageMergeTree.cpp | 1 - src/Storages/StorageReplicatedMergeTree.cpp | 3 +-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3e5dd709ab..fd4ff90f841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ * Added `FROM ` modifier for `SYSTEM SYNC REPLICA LIGHTWEIGHT` query. With the `FROM` modifier ensures we wait for fetches and drop-ranges only for the specified source replicas, as well as any replica not in zookeeper or with an empty source_replica. [#58393](https://github.com/ClickHouse/ClickHouse/pull/58393) ([Jayme Bird](https://github.com/jaymebrd)). * Added setting `update_insert_deduplication_token_in_dependent_materialized_views`. This setting allows to update insert deduplication token with table identifier during insert in dependent materialized views. Closes [#59165](https://github.com/ClickHouse/ClickHouse/issues/59165). [#59238](https://github.com/ClickHouse/ClickHouse/pull/59238) ([Maksim Kita](https://github.com/kitaisreal)). * Added statement `SYSTEM RELOAD ASYNCHRONOUS METRICS` which updates the asynchronous metrics. Mostly useful for testing and development. [#53710](https://github.com/ClickHouse/ClickHouse/pull/53710) ([Robert Schulze](https://github.com/rschu1ze)). +* Attach parts from a different disk `ALTER TABLE destination ATTACH PARTITION tuple() FROM source` where source is an [instant table](https://github.com/ClickHouse/web-tables-demo). [#60112](https://github.com/ClickHouse/ClickHouse/pull/60112)([Unalian](https://github.com/Unalian)). #### Performance Improvement * Coordination for parallel replicas is rewritten for better parallelism and cache locality. It has been tested for linear scalability on hundreds of replicas. It also got support for reading in order. [#57968](https://github.com/ClickHouse/ClickHouse/pull/57968) ([Nikita Taranov](https://github.com/nickitat)). diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 64787d3509b..be1346e0ea2 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -8,6 +8,7 @@ #include #include #include +#include "Common/logger_useful.h" #include #include #include @@ -7170,7 +7171,9 @@ std::pair MergeTreeData::cloneAn } } if (!copy_successful) - throw; + { + LOG_FATAL(&Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail"); + } } @@ -7301,8 +7304,9 @@ std::pair MergeTreeData::cloneAn } } if (!copy_successful) - throw; - + { + LOG_FATAL( &Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail."); + } if (params.metadata_version_to_write.has_value()) { chassert(!params.keep_metadata_version); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 47684925182..0748ac2dbdf 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2104,7 +2104,6 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con } else { - clone_params.copy_instead_of_hardlink = true; auto [dst_part, part_lock] = cloneAndLoadDataPartOnOtherDisk( src_part, TMP_PREFIX, diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index df261053360..2460d2704c4 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7933,7 +7933,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) if (disk->getName() == src_part->getDataPartStorage().getDiskName()) on_same_disk = true; - if (on_same_disk) + if (on_same_disk && !clone_params.copy_instead_of_hardlink) { auto [dst_part, part_lock] = cloneAndLoadDataPartOnSameDisk( src_part, @@ -7948,7 +7948,6 @@ void StorageReplicatedMergeTree::replacePartitionFrom( } else { - clone_params.copy_instead_of_hardlink = true; auto [dst_part, part_lock] = cloneAndLoadDataPartOnOtherDisk( src_part, TMP_PREFIX, From fc3ebe007b3b5dc905ecbd63ed402547a1cde3a5 Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 20 Feb 2024 20:54:32 +0800 Subject: [PATCH 0036/1165] [fix] rm whitespaces --- src/Storages/MergeTree/MergeTreeData.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index be1346e0ea2..18bb0966bfc 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7171,9 +7171,7 @@ std::pair MergeTreeData::cloneAn } } if (!copy_successful) - { LOG_FATAL(&Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail"); - } } @@ -7304,9 +7302,7 @@ std::pair MergeTreeData::cloneAn } } if (!copy_successful) - { LOG_FATAL( &Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail."); - } if (params.metadata_version_to_write.has_value()) { chassert(!params.keep_metadata_version); From f829a97d9130de5609e07e237b9486847422bc8c Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 20 Feb 2024 21:08:24 +0800 Subject: [PATCH 0037/1165] [fix] rm whitespaces --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 18bb0966bfc..849ceb1b66d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7302,7 +7302,7 @@ std::pair MergeTreeData::cloneAn } } if (!copy_successful) - LOG_FATAL( &Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail."); + LOG_FATAL(&Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail."); if (params.metadata_version_to_write.has_value()) { chassert(!params.keep_metadata_version); From 28282eee91add78e5b18202bd38566d1d3797083 Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 20 Feb 2024 21:37:09 +0800 Subject: [PATCH 0038/1165] [fix] Add description in partition.md --- docs/en/sql-reference/statements/alter/partition.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index 114b8d5ffe3..277e174bb05 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -116,6 +116,8 @@ For the query to run successfully, the following conditions must be met: - Both tables must have the same indices and projections. - Both tables must have the same storage policy. +If both tables have the same storage policy, use hardlink to attach partition. Otherwise, use copying the data to attach partition. + ## REPLACE PARTITION ``` sql From 1731a5a8afba5a48ce01cea20e0cdc1f91316841 Mon Sep 17 00:00:00 2001 From: unashi Date: Wed, 21 Feb 2024 10:55:32 +0800 Subject: [PATCH 0039/1165] [improve]change the integration test test_multiple_disks::test_move_across_policies_not_work to test_move_across_policies_work_for_attach_not_work_for_move --- tests/integration/test_multiple_disks/test.py | 36 +++++++------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index fdd81284b2a..9584ace7f45 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -5,6 +5,7 @@ import string import threading import time from multiprocessing.dummy import Pool +from helpers.test_tools import assert_eq_with_retry import pytest from helpers.client import QueryRuntimeException @@ -1745,9 +1746,9 @@ def test_move_while_merge(start_cluster): node1.query(f"DROP TABLE IF EXISTS {name} SYNC") -def test_move_across_policies_does_not_work(start_cluster): +def test_move_across_policies_work_for_attach_not_work_for_move(start_cluster): try: - name = "test_move_across_policies_does_not_work" + name = "test_move_across_policies_work_for_attach_not_work_for_move" node1.query( """ @@ -1783,25 +1784,18 @@ def test_move_across_policies_does_not_work(start_cluster): except QueryRuntimeException: """All parts of partition 'all' are already on disk 'jbod2'.""" - with pytest.raises( - QueryRuntimeException, - match=".*because disk does not belong to storage policy.*", - ): - node1.query( - """ALTER TABLE {name}2 ATTACH PARTITION tuple() FROM {name}""".format( - name=name - ) + node1.query( + """ALTER TABLE {name}2 ATTACH PARTITION tuple() FROM {name}""".format( + name=name ) - - with pytest.raises( - QueryRuntimeException, - match=".*because disk does not belong to storage policy.*", - ): + ) + assert_eq_with_retry( + node1, + """SELECT * FROM {name}2""".format(name=name), node1.query( - """ALTER TABLE {name}2 REPLACE PARTITION tuple() FROM {name}""".format( - name=name - ) - ) + """SELECT * FROM {name}""".format(name=name), + ), + ) with pytest.raises( QueryRuntimeException, @@ -1813,10 +1807,6 @@ def test_move_across_policies_does_not_work(start_cluster): ) ) - assert node1.query( - """SELECT * FROM {name}""".format(name=name) - ).splitlines() == ["1"] - finally: node1.query(f"DROP TABLE IF EXISTS {name} SYNC") node1.query(f"DROP TABLE IF EXISTS {name}2 SYNC") From 835b47519a7c575d70542e5a37c97dbf5a2b25f9 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 14 Feb 2024 00:44:38 +0100 Subject: [PATCH 0040/1165] impl --- src/Backups/BackupIO_S3.cpp | 1 + src/Coordination/KeeperSnapshotManagerS3.cpp | 1 + src/Disks/ObjectStorages/S3/diskSettings.cpp | 1 + src/IO/S3/Client.cpp | 19 +++++++++--- src/IO/S3/Client.h | 6 ++++ src/IO/S3/Requests.h | 32 ++++++++++++++++++-- src/IO/S3/URI.cpp | 18 +++++------ src/IO/S3/tests/gtest_aws_s3_client.cpp | 25 ++++++++++++++- src/IO/WriteBufferFromS3.cpp | 15 ++++++++- src/IO/WriteBufferFromS3.h | 1 + src/IO/tests/gtest_s3_uri.cpp | 8 +++++ src/IO/tests/gtest_writebuffer_s3.cpp | 21 +++++++------ src/Storages/StorageS3.cpp | 1 + 13 files changed, 119 insertions(+), 30 deletions(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 9359602a651..2063af2061c 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -73,6 +73,7 @@ namespace .use_virtual_addressing = s3_uri.is_virtual_hosted_style, .disable_checksum = local_settings.s3_disable_checksum, .gcs_issue_compose_request = context->getConfigRef().getBool("s3.gcs_issue_compose_request", false), + .is_s3express_bucket = S3::isS3ExpressEndpoint(s3_uri.endpoint), }; return S3::ClientFactory::instance().create( diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index 0337a564660..9779a041095 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -103,6 +103,7 @@ void KeeperSnapshotManagerS3::updateS3Configuration(const Poco::Util::AbstractCo .use_virtual_addressing = new_uri.is_virtual_hosted_style, .disable_checksum = false, .gcs_issue_compose_request = false, + .is_s3express_bucket = S3::isS3ExpressEndpoint(new_uri.endpoint), }; auto client = S3::ClientFactory::instance().create( diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 4fd4b17aabe..b8688cd3de6 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -97,6 +97,7 @@ std::unique_ptr getClient( .use_virtual_addressing = uri.is_virtual_hosted_style, .disable_checksum = local_settings.s3_disable_checksum, .gcs_issue_compose_request = config.getBool("s3.gcs_issue_compose_request", false), + .is_s3express_bucket = S3::isS3ExpressEndpoint(endpoint), }; return S3::ClientFactory::instance().create( diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 1b6b245b89a..a75d41df3d1 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -304,6 +304,9 @@ Model::HeadObjectOutcome Client::HeadObject(HeadObjectRequest & request) const request.setApiMode(api_mode); + if (isS3ExpressBucket()) + request.setIsS3ExpressBucket(); + addAdditionalAMZHeadersToCanonicalHeadersList(request, client_configuration.extra_headers); if (auto region = getRegionForBucket(bucket); !region.empty()) @@ -530,7 +533,11 @@ Client::doRequest(RequestType & request, RequestFn request_fn) const addAdditionalAMZHeadersToCanonicalHeadersList(request, client_configuration.extra_headers); const auto & bucket = request.GetBucket(); request.setApiMode(api_mode); - if (client_settings.disable_checksum) + + /// We have to use checksums for S3Express buckets, so the order of checks should be the following + if (client_settings.is_s3express_bucket) + request.setIsS3ExpressBucket(); + else if (client_settings.disable_checksum) request.disableChecksum(); if (auto region = getRegionForBucket(bucket); !region.empty()) @@ -915,9 +922,9 @@ std::unique_ptr ClientFactory::create( // NOLINT std::move(sse_kms_config), credentials_provider, client_configuration, // Client configuration. - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - client_settings - ); + client_settings.is_s3express_bucket ? Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent + : Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + client_settings); } PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT @@ -956,6 +963,10 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT return config; } +bool isS3ExpressEndpoint(const std::string & endpoint) +{ + return endpoint.contains("s3express"); +} } } diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 8da21bd2c2c..c7bc727bf32 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -92,6 +92,8 @@ private: std::unordered_map> client_caches; }; +bool isS3ExpressEndpoint(const std::string & endpoint); + struct ClientSettings { bool use_virtual_addressing; @@ -107,6 +109,7 @@ struct ClientSettings /// Ability to enable it preserved since likely it is required for old /// files. bool gcs_issue_compose_request; + bool is_s3express_bucket; }; /// Client that improves the client from the AWS SDK @@ -208,6 +211,9 @@ public: const std::shared_ptr& httpRequest) const override; bool supportsMultiPartCopy() const; + + bool isS3ExpressBucket() const { return client_settings.is_s3express_bucket; } + private: friend struct ::MockS3::Client; diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index bfb94a5a67e..6f82a0f39d3 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -21,12 +21,32 @@ #include #include #include +#include +#include + +#include namespace DB::S3 { namespace Model = Aws::S3::Model; +/// Used only for S3Express +namespace RequestChecksum +{ +inline void setPartChecksum(Model::CompletedPart & part, const std::string & checksum) +{ + part.SetChecksumCRC32(checksum); +} + +template +inline void setChecksumAlgorithm(R & request) +{ + if constexpr (requires { request.SetChecksumAlgorithm(Model::ChecksumAlgorithm::CRC32); }) + request.SetChecksumAlgorithm(Model::ChecksumAlgorithm::CRC32); +} +}; + template class ExtendedRequest : public BaseRequest { @@ -49,11 +69,13 @@ public: Aws::String GetChecksumAlgorithmName() const override { + chassert(!is_s3express_bucket || checksum); + /// Return empty string is enough to disable checksums (see /// AWSClient::AddChecksumToRequest [1] for more details). /// /// [1]: https://github.com/aws/aws-sdk-cpp/blob/b0ee1c0d336dbb371c34358b68fba6c56aae2c92/src/aws-cpp-sdk-core/source/client/AWSClient.cpp#L783-L839 - if (!checksum) + if (!is_s3express_bucket && !checksum) return ""; return BaseRequest::GetChecksumAlgorithmName(); } @@ -84,9 +106,12 @@ public: } /// Disable checksum to avoid extra read of the input stream - void disableChecksum() const + void disableChecksum() const { checksum = false; } + + void setIsS3ExpressBucket() { - checksum = false; + is_s3express_bucket = true; + RequestChecksum::setChecksumAlgorithm(*this); } protected: @@ -94,6 +119,7 @@ protected: mutable std::optional uri_override; mutable ApiMode api_mode{ApiMode::AWS}; mutable bool checksum = true; + bool is_s3express_bucket = false; }; class CopyObjectRequest : public ExtendedRequest diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 23f59420bfe..062d3b80850 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -35,7 +35,7 @@ URI::URI(const std::string & uri_) /// Case when bucket name represented in domain name of S3 URL. /// E.g. (https://bucket-name.s3.Region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access - static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3|cos|obs|oss|eos)([.\-][a-z0-9\-.:]+))"); + static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3express[\-a-z0-9]+|s3|cos|obs|oss|eos)([.\-][a-z0-9\-.:]+))"); /// Case when bucket name and key represented in path of S3 URL. /// E.g. (https://s3.Region.amazonaws.com/bucket-name/key) @@ -43,6 +43,7 @@ URI::URI(const std::string & uri_) static const RE2 path_style_pattern("^/([^/]*)/(.*)"); static constexpr auto S3 = "S3"; + static constexpr auto S3EXPRESS = "S3EXPRESS"; static constexpr auto COSN = "COSN"; static constexpr auto COS = "COS"; static constexpr auto OBS = "OBS"; @@ -115,21 +116,16 @@ URI::URI(const std::string & uri_) } boost::to_upper(name); - if (name != S3 && name != COS && name != OBS && name != OSS && name != EOS) + /// For S3Express it will look like s3express-eun1-az1, i.e. contain region and AZ info + if (name != S3 && !name.starts_with(S3EXPRESS) && name != COS && name != OBS && name != OSS && name != EOS) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Object storage system name is unrecognized in virtual hosted style S3 URI: {}", quoteString(name)); - if (name == S3) - storage_name = name; - else if (name == OBS) - storage_name = OBS; - else if (name == OSS) - storage_name = OSS; - else if (name == EOS) - storage_name = EOS; - else + if (name == COS || name == COSN) storage_name = COSN; + else + storage_name = name; } else if (re2::RE2::PartialMatch(uri.getPath(), path_style_pattern, &bucket, &key)) { diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index 33917314bca..8edbe12a22f 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -110,7 +110,8 @@ void testServerSideEncryption( bool disable_checksum, String server_side_encryption_customer_key_base64, DB::S3::ServerSideEncryptionKMSConfig sse_kms_config, - String expected_headers) + String expected_headers, + bool is_s3express_bucket = false) { TestPocoHTTPServer http; @@ -144,6 +145,7 @@ void testServerSideEncryption( .use_virtual_addressing = uri.is_virtual_hosted_style, .disable_checksum = disable_checksum, .gcs_issue_compose_request = false, + .is_s3express_bucket = is_s3express_bucket, }; std::shared_ptr client = DB::S3::ClientFactory::instance().create( @@ -295,4 +297,25 @@ TEST(IOTestAwsS3Client, AppendExtraSSEKMSHeadersWrite) "x-amz-server-side-encryption-context: arn:aws:s3:::bucket_ARN\n"); } +TEST(IOTestAwsS3Client, ChecksumHeaderIsPresentForS3Express) +{ + /// See https://github.com/ClickHouse/ClickHouse/pull/19748 + testServerSideEncryption( + doWriteRequest, + /* disable_checksum= */ true, + "", + {}, + "authorization: ... SignedHeaders=" + "amz-sdk-invocation-id;" + "amz-sdk-request;" + "content-length;" + "content-type;" + "host;" + "x-amz-checksum-crc32;" + "x-amz-content-sha256;" + "x-amz-date;" + "x-amz-sdk-checksum-algorithm, ...\n", + /*is_s3express_bucket=*/true); +} + #endif diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 5bb01050591..a162992278f 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -18,7 +18,9 @@ #include #include +#include #include +#include #include @@ -456,6 +458,14 @@ S3::UploadPartRequest WriteBufferFromS3::getUploadRequest(size_t part_number, Pa /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 req.SetContentType("binary/octet-stream"); + /// Checksums need to be provided on CompleteMultipartUpload requests, so we calculate then manually and store in multipart_checksums + if (client_ptr->isS3ExpressBucket()) + { + chassert(req.GetChecksumAlgorithm() == Aws::S3::Model::ChecksumAlgorithm::CRC32); + req.SetChecksumCRC32(Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateCRC32(*(req.GetBody())))); + multipart_checksums.push_back(req.GetChecksumCRC32()); + } + return req; } @@ -575,7 +585,10 @@ void WriteBufferFromS3::completeMultipartUpload() for (size_t i = 0; i < multipart_tags.size(); ++i) { Aws::S3::Model::CompletedPart part; - multipart_upload.AddParts(part.WithETag(multipart_tags[i]).WithPartNumber(static_cast(i + 1))); + part.WithETag(multipart_tags[i]).WithPartNumber(static_cast(i + 1)); + if (!multipart_checksums.empty()) + S3::RequestChecksum::setPartChecksum(part, multipart_checksums.at(i)); + multipart_upload.AddParts(part); } req.SetMultipartUpload(multipart_upload); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 5dc269990a1..148cd27f854 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -100,6 +100,7 @@ private: /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts. String multipart_upload_id; std::deque multipart_tags; + std::deque multipart_checksums; bool multipart_upload_finished = false; /// Track that prefinalize() is called only once diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index c088e41f1e8..5bf0dfb962d 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -162,6 +162,14 @@ TEST(S3UriTest, validPatterns) ASSERT_EQ("", uri.version_id); ASSERT_EQ(false, uri.is_virtual_hosted_style); } + { + S3::URI uri("https://test-perf-bucket--eun1-az1--x-s3.s3express-eun1-az1.eu-north-1.amazonaws.com/test.csv"); + ASSERT_EQ("https://s3express-eun1-az1.eu-north-1.amazonaws.com", uri.endpoint); + ASSERT_EQ("test-perf-bucket--eun1-az1--x-s3", uri.bucket); + ASSERT_EQ("test.csv", uri.key); + ASSERT_EQ("", uri.version_id); + ASSERT_EQ(true, uri.is_virtual_hosted_style); + } } TEST_P(S3UriTest, invalidPatterns) diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index ae00bb2e9e2..d9cb486c09e 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -205,16 +205,17 @@ struct Client : DB::S3::Client { explicit Client(std::shared_ptr mock_s3_store) : DB::S3::Client( - 100, - DB::S3::ServerSideEncryptionKMSConfig(), - std::make_shared("", ""), - GetClientConfiguration(), - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - DB::S3::ClientSettings{ - .use_virtual_addressing = true, - .disable_checksum= false, - .gcs_issue_compose_request = false, - }) + 100, + DB::S3::ServerSideEncryptionKMSConfig(), + std::make_shared("", ""), + GetClientConfiguration(), + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + DB::S3::ClientSettings{ + .use_virtual_addressing = true, + .disable_checksum = false, + .gcs_issue_compose_request = false, + .is_s3express_bucket = false, + }) , store(mock_s3_store) {} diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 2d8ef3df1c8..044a1ca5362 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1427,6 +1427,7 @@ void StorageS3::Configuration::connect(const ContextPtr & context) .use_virtual_addressing = url.is_virtual_hosted_style, .disable_checksum = local_settings.s3_disable_checksum, .gcs_issue_compose_request = context->getConfigRef().getBool("s3.gcs_issue_compose_request", false), + .is_s3express_bucket = S3::isS3ExpressEndpoint(url.endpoint), }; auto credentials = Aws::Auth::AWSCredentials(auth_settings.access_key_id, auth_settings.secret_access_key, auth_settings.session_token); From 7ac453ab4c2f7162c5dc25f29eaf396b670357a6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 27 Feb 2024 23:06:14 +0800 Subject: [PATCH 0041/1165] Revert "Merge pull request #60436 from nickitat/revert_56864" This reverts commit 8719a601fac863a2c484bcf97339aecdf9e73c5f, reversing changes made to 657857f9828eb46867197c6f7bc8c2444ab1cc3d. --- src/Backups/BackupSettings.cpp | 2 + src/Backups/BackupSettings.h | 6 + src/Common/ErrorCodes.cpp | 1 + src/Interpreters/MutationsInterpreter.cpp | 14 +- src/Interpreters/MutationsInterpreter.h | 1 + .../Optimizations/projectionsCommon.cpp | 2 +- .../MergeTree/DataPartStorageOnDiskBase.cpp | 35 +- .../MergeTree/DataPartStorageOnDiskBase.h | 4 +- src/Storages/MergeTree/IDataPartStorage.h | 4 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 62 +- src/Storages/MergeTree/IMergeTreeDataPart.h | 17 +- src/Storages/MergeTree/MergeTask.cpp | 5 +- src/Storages/MergeTree/MergeTreeData.cpp | 37 +- src/Storages/MergeTree/MergeTreeData.h | 9 +- .../MergeTree/MergeTreeDataPartChecksum.h | 2 + src/Storages/MergeTree/MutateTask.cpp | 14 +- .../ReplicatedMergeTreePartCheckThread.cpp | 38 +- .../ReplicatedMergeTreePartCheckThread.h | 4 +- src/Storages/MergeTree/checkDataPart.cpp | 71 ++- src/Storages/MergeTree/checkDataPart.h | 4 +- src/Storages/StorageMergeTree.cpp | 5 +- src/Storages/StorageReplicatedMergeTree.cpp | 3 +- .../System/StorageSystemProjectionParts.cpp | 34 +- .../test_broken_projections/__init__.py | 0 .../config.d/backups.xml | 13 + .../test_broken_projections/test.py | 576 ++++++++++++++++++ .../02117_show_create_table_system.reference | 3 + 27 files changed, 907 insertions(+), 59 deletions(-) create mode 100644 tests/integration/test_broken_projections/__init__.py create mode 100644 tests/integration/test_broken_projections/config.d/backups.xml create mode 100644 tests/integration/test_broken_projections/test.py diff --git a/src/Backups/BackupSettings.cpp b/src/Backups/BackupSettings.cpp index 68d825e9468..51d713f03e1 100644 --- a/src/Backups/BackupSettings.cpp +++ b/src/Backups/BackupSettings.cpp @@ -32,6 +32,8 @@ namespace ErrorCodes M(UInt64, shard_num) \ M(UInt64, replica_num) \ M(Bool, check_parts) \ + M(Bool, check_projection_parts) \ + M(Bool, allow_backup_broken_projections) \ M(Bool, internal) \ M(String, host_id) \ M(OptionalUUID, backup_uuid) diff --git a/src/Backups/BackupSettings.h b/src/Backups/BackupSettings.h index f26b992b348..ec430905f51 100644 --- a/src/Backups/BackupSettings.h +++ b/src/Backups/BackupSettings.h @@ -62,6 +62,12 @@ struct BackupSettings /// Check checksums of the data parts before writing them to a backup. bool check_parts = true; + /// Check checksums of the projection data parts before writing them to a backup. + bool check_projection_parts = true; + + /// Allow to create backup with broken projections. + bool allow_backup_broken_projections = false; + /// Internal, should not be specified by user. /// Whether this backup is a part of a distributed backup created by BACKUP ON CLUSTER. bool internal = false; diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index ca00f2fd513..1ce8997e928 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -592,6 +592,7 @@ M(710, FAULT_INJECTED) \ M(711, FILECACHE_ACCESS_DENIED) \ M(712, TOO_MANY_MATERIALIZED_VIEWS) \ + M(713, BROKEN_PROJECTION) \ M(714, UNEXPECTED_CLUSTER) \ M(715, CANNOT_DETECT_FORMAT) \ M(716, CANNOT_FORGET_PARTITION) \ diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index a3d1b84fdc1..502b961ced8 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -342,6 +342,11 @@ bool MutationsInterpreter::Source::hasProjection(const String & name) const return part && part->hasProjection(name); } +bool MutationsInterpreter::Source::hasBrokenProjection(const String & name) const +{ + return part && part->hasBrokenProjection(name); +} + bool MutationsInterpreter::Source::isCompactPart() const { return part && part->getType() == MergeTreeDataPartType::Compact; @@ -807,7 +812,7 @@ void MutationsInterpreter::prepare(bool dry_run) { mutation_kind.set(MutationKind::MUTATE_INDEX_STATISTIC_PROJECTION); const auto & projection = projections_desc.get(command.projection_name); - if (!source.hasProjection(projection.name)) + if (!source.hasProjection(projection.name) || source.hasBrokenProjection(projection.name)) { for (const auto & column : projection.required_columns) dependencies.emplace(column, ColumnDependency::PROJECTION); @@ -994,6 +999,13 @@ void MutationsInterpreter::prepare(bool dry_run) if (!source.hasProjection(projection.name)) continue; + /// Always rebuild broken projections. + if (source.hasBrokenProjection(projection.name)) + { + materialized_projections.insert(projection.name); + continue; + } + if (need_rebuild_projections) { materialized_projections.insert(projection.name); diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index eda94190185..4c35ec34b58 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -126,6 +126,7 @@ public: bool materializeTTLRecalculateOnly() const; bool hasSecondaryIndex(const String & name) const; bool hasProjection(const String & name) const; + bool hasBrokenProjection(const String & name) const; bool isCompactPart() const; void read( diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp index 8333f5e857b..3009460a468 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp @@ -223,7 +223,7 @@ bool analyzeProjectionCandidate( { const auto & created_projections = part_with_ranges.data_part->getProjectionParts(); auto it = created_projections.find(candidate.projection->name); - if (it != created_projections.end()) + if (it != created_projections.end() && !it->second->is_broken) { projection_parts.push_back(it->second); } diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index 5210d14f3d0..e31d991ef09 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -335,7 +335,9 @@ void DataPartStorageOnDiskBase::backup( const ReadSettings & read_settings, bool make_temporary_hard_links, BackupEntries & backup_entries, - TemporaryFilesOnDisks * temp_dirs) const + TemporaryFilesOnDisks * temp_dirs, + bool is_projection_part, + bool allow_backup_broken_projection) const { fs::path part_path_on_disk = fs::path{root_path} / part_dir; fs::path part_path_in_backup = fs::path{path_in_backup} / part_dir; @@ -377,7 +379,7 @@ void DataPartStorageOnDiskBase::backup( bool copy_encrypted = !backup_settings.decrypt_files_from_encrypted_disks; - for (const auto & filepath : files_to_backup) + auto backup_file = [&](const String & filepath) { auto filepath_on_disk = part_path_on_disk / filepath; auto filepath_in_backup = part_path_in_backup / filepath; @@ -385,8 +387,10 @@ void DataPartStorageOnDiskBase::backup( if (files_without_checksums.contains(filepath)) { backup_entries.emplace_back(filepath_in_backup, std::make_unique(disk, filepath_on_disk, read_settings, copy_encrypted)); - continue; + return; } + else if (is_projection_part && allow_backup_broken_projection && !disk->exists(filepath_on_disk)) + return; if (make_temporary_hard_links) { @@ -411,6 +415,31 @@ void DataPartStorageOnDiskBase::backup( backup_entry = wrapBackupEntryWith(std::move(backup_entry), temp_dir_owner); backup_entries.emplace_back(filepath_in_backup, std::move(backup_entry)); + }; + + auto * log = &Poco::Logger::get("DataPartStorageOnDiskBase::backup"); + + for (const auto & filepath : files_to_backup) + { + if (is_projection_part && allow_backup_broken_projection) + { + try + { + backup_file(filepath); + } + catch (Exception & e) + { + if (e.code() != ErrorCodes::FILE_DOESNT_EXIST) + throw; + + LOG_ERROR(log, "Cannot backup file {} of projection part {}. Will try to ignore it", filepath, part_dir); + continue; + } + } + else + { + backup_file(filepath); + } } } diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h index 52dc850c7fd..75bf3d6f93c 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h @@ -58,7 +58,9 @@ public: const ReadSettings & read_settings, bool make_temporary_hard_links, BackupEntries & backup_entries, - TemporaryFilesOnDisks * temp_dirs) const override; + TemporaryFilesOnDisks * temp_dirs, + bool is_projection_part, + bool allow_backup_broken_projection) const override; MutableDataPartStoragePtr freeze( const std::string & to, diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 5899ef58cd5..d06d9791a53 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -223,7 +223,9 @@ public: const ReadSettings & read_settings, bool make_temporary_hard_links, BackupEntries & backup_entries, - TemporaryFilesOnDisks * temp_dirs) const = 0; + TemporaryFilesOnDisks * temp_dirs, + bool is_projection_part, + bool allow_backup_broken_projection) const = 0; /// Creates hardlinks into 'to/dir_path' for every file in data part. /// Callback is called after hardlinks are created, but before 'delete-on-destroy.txt' marker is removed. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index e06ea5e560c..11ede661f78 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -699,13 +699,14 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks calculateColumnsAndSecondaryIndicesSizesOnDisk(); loadRowsCount(); /// Must be called after loadIndexGranularity() as it uses the value of `index_granularity`. loadPartitionAndMinMaxIndex(); + bool has_broken_projections = false; if (!parent_part) { loadTTLInfos(); - loadProjections(require_columns_checksums, check_consistency, false /* if_not_loaded */); + loadProjections(require_columns_checksums, check_consistency, has_broken_projections, false /* if_not_loaded */); } - if (check_consistency) + if (check_consistency && !has_broken_projections) checkConsistency(require_columns_checksums); loadDefaultCompressionCodec(); @@ -770,7 +771,7 @@ void IMergeTreeDataPart::addProjectionPart( projection_parts[projection_name] = std::move(projection_part); } -void IMergeTreeDataPart::loadProjections(bool require_columns_checksums, bool check_consistency, bool if_not_loaded) +void IMergeTreeDataPart::loadProjections(bool require_columns_checksums, bool check_consistency, bool & has_broken_projection, bool if_not_loaded) { auto metadata_snapshot = storage.getInMemoryMetadataPtr(); for (const auto & projection : metadata_snapshot->projections) @@ -787,10 +788,34 @@ void IMergeTreeDataPart::loadProjections(bool require_columns_checksums, bool ch else { auto part = getProjectionPartBuilder(projection.name).withPartFormatFromDisk().build(); - part->loadColumnsChecksumsIndexes(require_columns_checksums, check_consistency); + + try + { + part->loadColumnsChecksumsIndexes(require_columns_checksums, check_consistency); + } + catch (...) + { + if (isRetryableException(std::current_exception())) + throw; + + auto message = getCurrentExceptionMessage(true); + LOG_ERROR(&Poco::Logger::get("IMergeTreeDataPart"), + "Cannot load projection {}, will consider it broken. Reason: {}", projection.name, message); + + has_broken_projection = true; + part->setBrokenReason(message, getCurrentExceptionCode()); + } + addProjectionPart(projection.name, std::move(part)); } } + else if (checksums.has(path)) + { + auto part = getProjectionPartBuilder(projection.name).withPartFormatFromDisk().build(); + part->setBrokenReason("Projection directory " + path + " does not exist while loading projections", ErrorCodes::NO_FILE_IN_DATA_PART); + addProjectionPart(projection.name, std::move(part)); + has_broken_projection = true; + } } } @@ -1189,7 +1214,8 @@ void IMergeTreeDataPart::loadChecksums(bool require) /// Check the data while we are at it. LOG_WARNING(storage.log, "Checksums for part {} not found. Will calculate them from data on disk.", name); - checksums = checkDataPart(shared_from_this(), false); + bool noop; + checksums = checkDataPart(shared_from_this(), false, noop, /* is_cancelled */[]{ return false; }, /* throw_on_broken_projection */false); writeChecksums(checksums, {}); bytes_on_disk = checksums.getTotalSizeOnDisk(); @@ -2196,6 +2222,32 @@ std::optional IMergeTreeDataPart::getStreamNameForColumn( return getStreamNameOrHash(stream_name, extension, storage_); } +void IMergeTreeDataPart::markProjectionPartAsBroken(const String & projection_name, const String & message, int code) const +{ + auto it = projection_parts.find(projection_name); + if (it == projection_parts.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no projection part '{}'", projection_name); + it->second->setBrokenReason(message, code); +} + +bool IMergeTreeDataPart::hasBrokenProjection(const String & projection_name) const +{ + auto it = projection_parts.find(projection_name); + if (it == projection_parts.end()) + return false; + return it->second->is_broken; +} + +void IMergeTreeDataPart::setBrokenReason(const String & message, int code) const +{ + std::lock_guard lock(broken_reason_mutex); + if (is_broken) + return; + is_broken = true; + exception = message; + exception_code = code; +} + bool isCompactPart(const MergeTreeDataPartPtr & data_part) { return (data_part && data_part->getType() == MergeTreeDataPartType::Compact); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 91c559d30c8..0d7acfab891 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -259,6 +259,12 @@ public: /// Frozen by ALTER TABLE ... FREEZE ... It is used for information purposes in system.parts table. mutable std::atomic is_frozen {false}; + /// If it is a projection part, it can be broken sometimes. + mutable std::atomic is_broken {false}; + mutable std::string exception; + mutable int exception_code = 0; + mutable std::mutex broken_reason_mutex; + /// Indicates that the part was marked Outdated by PartCheckThread because the part was not committed to ZooKeeper mutable bool is_unexpected_local_part = false; @@ -418,9 +424,16 @@ public: void addProjectionPart(const String & projection_name, std::shared_ptr && projection_part); + void markProjectionPartAsBroken(const String & projection_name, const String & message, int code) const; + bool hasProjection(const String & projection_name) const { return projection_parts.contains(projection_name); } - void loadProjections(bool require_columns_checksums, bool check_consistency, bool if_not_loaded = false); + bool hasBrokenProjection(const String & projection_name) const; + + /// Return true, if all projections were loaded successfully and none was marked as broken. + void loadProjections(bool require_columns_checksums, bool check_consistency, bool & has_broken_projection, bool if_not_loaded = false); + + void setBrokenReason(const String & message, int code) const; /// Return set of metadata file names without checksums. For example, /// columns.txt or checksums.txt itself. @@ -580,7 +593,7 @@ protected: const IMergeTreeDataPart * parent_part; String parent_part_name; - std::map> projection_parts; + mutable std::map> projection_parts; mutable PartMetadataManagerPtr metadata_manager; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index df64ae33713..e6ae63da7e3 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -731,8 +731,9 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c MergeTreeData::DataPartsVector projection_parts; for (const auto & part : global_ctx->future_part->parts) { - auto it = part->getProjectionParts().find(projection.name); - if (it != part->getProjectionParts().end()) + auto actual_projection_parts = part->getProjectionParts(); + auto it = actual_projection_parts.find(projection.name); + if (it != actual_projection_parts.end() && !it->second->is_broken) projection_parts.push_back(it->second); } if (projection_parts.size() < global_ctx->future_part->parts.size()) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6d5e486f6a1..babc593ff62 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5311,7 +5311,7 @@ MergeTreeData::PartsBackupEntries MergeTreeData::backupParts( if (hold_table_lock && !table_lock) table_lock = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); - if (backup_settings.check_parts) + if (backup_settings.check_projection_parts) part->checkConsistencyWithProjections(/* require_part_metadata= */ true); BackupEntries backup_entries_from_part; @@ -5323,7 +5323,8 @@ MergeTreeData::PartsBackupEntries MergeTreeData::backupParts( read_settings, make_temporary_hard_links, backup_entries_from_part, - &temp_dirs); + &temp_dirs, + false, false); auto projection_parts = part->getProjectionParts(); for (const auto & [projection_name, projection_part] : projection_parts) @@ -5336,7 +5337,9 @@ MergeTreeData::PartsBackupEntries MergeTreeData::backupParts( read_settings, make_temporary_hard_links, backup_entries_from_part, - &temp_dirs); + &temp_dirs, + projection_part->is_broken, + backup_settings.allow_backup_broken_projections); } if (hold_storage_and_part_ptrs) @@ -7825,21 +7828,39 @@ MovePartsOutcome MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & bool MergeTreeData::partsContainSameProjections(const DataPartPtr & left, const DataPartPtr & right, String & out_reason) { - if (left->getProjectionParts().size() != right->getProjectionParts().size()) + auto remove_broken_parts_from_consideration = [](auto & parts) + { + std::set broken_projection_parts; + for (const auto & [name, part] : parts) + { + if (part->is_broken) + broken_projection_parts.emplace(name); + } + for (const auto & name : broken_projection_parts) + parts.erase(name); + }; + + auto left_projection_parts = left->getProjectionParts(); + auto right_projection_parts = right->getProjectionParts(); + + remove_broken_parts_from_consideration(left_projection_parts); + remove_broken_parts_from_consideration(right_projection_parts); + + if (left_projection_parts.size() != right_projection_parts.size()) { out_reason = fmt::format( "Parts have different number of projections: {} in part '{}' and {} in part '{}'", - left->getProjectionParts().size(), + left_projection_parts.size(), left->name, - right->getProjectionParts().size(), + right_projection_parts.size(), right->name ); return false; } - for (const auto & [name, _] : left->getProjectionParts()) + for (const auto & [name, _] : left_projection_parts) { - if (!right->hasProjection(name)) + if (!right_projection_parts.contains(name)) { out_reason = fmt::format( "The part '{}' doesn't have projection '{}' while part '{}' does", right->name, name, left->name diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index ab265715688..c638505604f 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -468,8 +468,13 @@ public: struct ProjectionPartsVector { - DataPartsVector projection_parts; DataPartsVector data_parts; + + DataPartsVector projection_parts; + DataPartStateVector projection_parts_states; + + DataPartsVector broken_projection_parts; + DataPartStateVector broken_projection_parts_states; }; /// Returns a copy of the list so that the caller shouldn't worry about locks. @@ -484,7 +489,7 @@ public: const DataPartStates & affordable_states, DataPartStateVector * out_states = nullptr) const; /// Same as above but only returns projection parts ProjectionPartsVector getProjectionPartsVectorForInternalUsage( - const DataPartStates & affordable_states, DataPartStateVector * out_states = nullptr) const; + const DataPartStates & affordable_states, MergeTreeData::DataPartStateVector * out_states) const; /// Returns absolutely all parts (and snapshot of their states) diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h index 837b940e354..d4980a67a43 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -54,6 +54,8 @@ struct MergeTreeDataPartChecksums bool has(const String & file_name) const { return files.find(file_name) != files.end(); } + bool remove(const String & file_name) { return files.erase(file_name); } + bool empty() const { return files.empty(); } /// Checks that the set of columns and their checksums are the same. If not, throws an exception. diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 1c33f018a5d..6bacce9e2c5 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -573,7 +573,9 @@ static std::set getProjectionsToRecalculate( { bool need_recalculate = materialized_projections.contains(projection.name) - || (!is_full_part_storage && source_part->hasProjection(projection.name)); + || (!is_full_part_storage + && source_part->hasProjection(projection.name) + && !source_part->hasBrokenProjection(projection.name)); if (need_recalculate) projections_to_recalc.insert(&projection); @@ -917,7 +919,8 @@ void finalizeMutatedPart( new_data_part->modification_time = time(nullptr); /// Load rest projections which are hardlinked - new_data_part->loadProjections(false, false, true /* if_not_loaded */); + bool noop; + new_data_part->loadProjections(false, false, noop, true /* if_not_loaded */); /// All information about sizes is stored in checksums. /// It doesn't make sense to touch filesystem for sizes. @@ -1500,7 +1503,9 @@ private: bool need_recalculate = ctx->materialized_projections.contains(projection.name) - || (!is_full_part_storage && ctx->source_part->hasProjection(projection.name)); + || (!is_full_part_storage + && ctx->source_part->hasProjection(projection.name) + && !ctx->source_part->hasBrokenProjection(projection.name)); if (need_recalculate) { @@ -1637,8 +1642,9 @@ private: void finalize() { + bool noop; ctx->new_data_part->minmax_idx = std::move(ctx->minmax_idx); - ctx->new_data_part->loadProjections(false, false, true /* if_not_loaded */); + ctx->new_data_part->loadProjections(false, false, noop, true /* if_not_loaded */); ctx->mutating_executor.reset(); ctx->mutating_pipeline.reset(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 156c41563ec..bc0b4f73a31 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -63,7 +63,7 @@ void ReplicatedMergeTreePartCheckThread::enqueuePart(const String & name, time_t if (parts_set.contains(name)) return; - LOG_TRACE(log, "Enqueueing {} for check after after {}s", name, delay_to_check_seconds); + LOG_TRACE(log, "Enqueueing {} for check after {}s", name, delay_to_check_seconds); parts_queue.emplace_back(name, std::chrono::steady_clock::now() + std::chrono::seconds(delay_to_check_seconds)); parts_set.insert(name); task->schedule(); @@ -274,7 +274,7 @@ std::pair ReplicatedMergeTreePartCheckThread::findLo return std::make_pair(exists_in_zookeeper, part); } -ReplicatedCheckResult ReplicatedMergeTreePartCheckThread::checkPartImpl(const String & part_name) +ReplicatedCheckResult ReplicatedMergeTreePartCheckThread::checkPartImpl(const String & part_name, bool throw_on_broken_projection) { ReplicatedCheckResult result; auto [exists_in_zookeeper, part] = findLocalPart(part_name); @@ -341,6 +341,7 @@ ReplicatedCheckResult ReplicatedMergeTreePartCheckThread::checkPartImpl(const St /// before the ReplicatedMergeTreePartHeader was introduced. String part_path = storage.replica_path + "/parts/" + part_name; String part_znode = zookeeper->get(part_path); + bool is_broken_projection = false; try { @@ -362,8 +363,10 @@ ReplicatedCheckResult ReplicatedMergeTreePartCheckThread::checkPartImpl(const St checkDataPart( part, - true, - [this] { return need_stop.load(); }); + /* require_checksums */true, + is_broken_projection, + [this] { return need_stop.load(); }, + throw_on_broken_projection); if (need_stop) { @@ -382,14 +385,27 @@ ReplicatedCheckResult ReplicatedMergeTreePartCheckThread::checkPartImpl(const St if (isRetryableException(std::current_exception())) throw; - tryLogCurrentException(log, __PRETTY_FUNCTION__); + PreformattedMessage message; + if (is_broken_projection) + { + WriteBufferFromOwnString wb; + message = PreformattedMessage::create( + "Part {} has a broken projections. It will be ignored. Broken projections info: {}", + part_name, getCurrentExceptionMessage(false)); + LOG_DEBUG(log, message); + result.action = ReplicatedCheckResult::DoNothing; + } + else + { + tryLogCurrentException(log, __PRETTY_FUNCTION__); - auto message = PreformattedMessage::create("Part {} looks broken. Removing it and will try to fetch.", part_name); - LOG_ERROR(log, message); + message = PreformattedMessage::create("Part {} looks broken. Removing it and will try to fetch.", part_name); + LOG_ERROR(log, message); + result.action = ReplicatedCheckResult::TryFetchMissing; + } /// Part is broken, let's try to find it and fetch. result.status = {part_name, false, message}; - result.action = ReplicatedCheckResult::TryFetchMissing; return result; } @@ -419,12 +435,12 @@ ReplicatedCheckResult ReplicatedMergeTreePartCheckThread::checkPartImpl(const St } -CheckResult ReplicatedMergeTreePartCheckThread::checkPartAndFix(const String & part_name, std::optional * recheck_after) +CheckResult ReplicatedMergeTreePartCheckThread::checkPartAndFix(const String & part_name, std::optional * recheck_after, bool throw_on_broken_projection) { LOG_INFO(log, "Checking part {}", part_name); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecks); - ReplicatedCheckResult result = checkPartImpl(part_name); + ReplicatedCheckResult result = checkPartImpl(part_name, throw_on_broken_projection); switch (result.action) { case ReplicatedCheckResult::None: UNREACHABLE(); @@ -577,7 +593,7 @@ void ReplicatedMergeTreePartCheckThread::run() } std::optional recheck_after; - checkPartAndFix(selected->name, &recheck_after); + checkPartAndFix(selected->name, &recheck_after, /* throw_on_broken_projection */false); if (need_stop) return; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h index f2e26b3d324..9091f698546 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h @@ -65,9 +65,9 @@ public: size_t size() const; /// Check part by name - CheckResult checkPartAndFix(const String & part_name, std::optional * recheck_after = nullptr); + CheckResult checkPartAndFix(const String & part_name, std::optional * recheck_after = nullptr, bool throw_on_broken_projection = true); - ReplicatedCheckResult checkPartImpl(const String & part_name); + ReplicatedCheckResult checkPartImpl(const String & part_name, bool throw_on_broken_projection); std::unique_lock pausePartsCheck(); diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 8ae9b54b6e9..0b545beb116 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -43,6 +43,7 @@ namespace ErrorCodes extern const int NO_FILE_IN_DATA_PART; extern const int NETWORK_ERROR; extern const int SOCKET_TIMEOUT; + extern const int BROKEN_PROJECTION; } @@ -117,7 +118,9 @@ static IMergeTreeDataPart::Checksums checkDataPart( const NameSet & files_without_checksums, const ReadSettings & read_settings, bool require_checksums, - std::function is_cancelled) + std::function is_cancelled, + bool & is_broken_projection, + bool throw_on_broken_projection) { /** Responsibility: * - read list of columns from columns.txt; @@ -126,6 +129,7 @@ static IMergeTreeDataPart::Checksums checkDataPart( */ CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedChecks}; + Poco::Logger * log = &Poco::Logger::get("checkDataPart"); NamesAndTypesList columns_txt; @@ -275,17 +279,55 @@ static IMergeTreeDataPart::Checksums checkDataPart( } } + std::string broken_projections_message; for (const auto & [name, projection] : data_part->getProjectionParts()) { if (is_cancelled()) return {}; auto projection_file = name + ".proj"; - auto projection_checksums = checkDataPart( - projection, *data_part_storage.getProjection(projection_file), - projection->getColumns(), projection->getType(), - projection->getFileNamesWithoutChecksums(), - read_settings, require_checksums, is_cancelled); + if (!throw_on_broken_projection && projection->is_broken) + { + projections_on_disk.erase(projection_file); + checksums_txt.remove(projection_file); + } + + IMergeTreeDataPart::Checksums projection_checksums; + try + { + bool noop; + projection_checksums = checkDataPart( + projection, *data_part_storage.getProjection(projection_file), + projection->getColumns(), projection->getType(), + projection->getFileNamesWithoutChecksums(), + read_settings, require_checksums, is_cancelled, noop, /* throw_on_broken_projection */false); + } + catch (...) + { + if (isRetryableException(std::current_exception())) + throw; + + if (!projection->is_broken) + { + LOG_TEST(log, "Marking projection {} as broken ({})", name, projection_file); + projection->setBrokenReason(getCurrentExceptionMessage(false), getCurrentExceptionCode()); + } + + is_broken_projection = true; + if (throw_on_broken_projection) + { + if (!broken_projections_message.empty()) + broken_projections_message += "\n"; + + broken_projections_message += fmt::format( + "Part {} has a broken projection {} (error: {})", + data_part->name, name, getCurrentExceptionMessage(false)); + continue; + } + + projections_on_disk.erase(projection_file); + checksums_txt.remove(projection_file); + } checksums_data.files[projection_file] = IMergeTreeDataPart::Checksums::Checksum( projection_checksums.getTotalSizeOnDisk(), @@ -294,6 +336,11 @@ static IMergeTreeDataPart::Checksums checkDataPart( projections_on_disk.erase(projection_file); } + if (throw_on_broken_projection && !broken_projections_message.empty()) + { + throw Exception(ErrorCodes::BROKEN_PROJECTION, "{}", broken_projections_message); + } + if (require_checksums && !projections_on_disk.empty()) { throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART, @@ -321,7 +368,9 @@ IMergeTreeDataPart::Checksums checkDataPartInMemory(const DataPartInMemoryPtr & IMergeTreeDataPart::Checksums checkDataPart( MergeTreeData::DataPartPtr data_part, bool require_checksums, - std::function is_cancelled) + bool & is_broken_projection, + std::function is_cancelled, + bool throw_on_broken_projection) { if (auto part_in_memory = asInMemoryPart(data_part)) return checkDataPartInMemory(part_in_memory); @@ -363,7 +412,9 @@ IMergeTreeDataPart::Checksums checkDataPart( data_part->getFileNamesWithoutChecksums(), read_settings, require_checksums, - is_cancelled); + is_cancelled, + is_broken_projection, + throw_on_broken_projection); }; try @@ -377,7 +428,9 @@ IMergeTreeDataPart::Checksums checkDataPart( data_part->getFileNamesWithoutChecksums(), read_settings, require_checksums, - is_cancelled); + is_cancelled, + is_broken_projection, + throw_on_broken_projection); } catch (...) { diff --git a/src/Storages/MergeTree/checkDataPart.h b/src/Storages/MergeTree/checkDataPart.h index d0e48b6f80a..a01978f4efe 100644 --- a/src/Storages/MergeTree/checkDataPart.h +++ b/src/Storages/MergeTree/checkDataPart.h @@ -10,7 +10,9 @@ namespace DB IMergeTreeDataPart::Checksums checkDataPart( MergeTreeData::DataPartPtr data_part, bool require_checksums, - std::function is_cancelled = []{ return false; }); + bool & is_broken_projection, + std::function is_cancelled = []{ return false; }, + bool throw_on_broken_projection = false); bool isNotEnoughMemoryErrorCode(int code); bool isRetryableException(const std::exception_ptr exception_ptr); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index e15b308f084..d94a4ff14c0 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2315,11 +2315,12 @@ std::optional StorageMergeTree::checkDataNext(DataValidationTasksPt { /// If the checksums file is not present, calculate the checksums and write them to disk. static constexpr auto checksums_path = "checksums.txt"; + bool noop; if (part->isStoredOnDisk() && !part->getDataPartStorage().exists(checksums_path)) { try { - auto calculated_checksums = checkDataPart(part, false); + auto calculated_checksums = checkDataPart(part, false, noop, /* is_cancelled */[]{ return false; }, /* throw_on_broken_projection */true); calculated_checksums.checkEqual(part->checksums, true); auto & part_mutable = const_cast(*part); @@ -2340,7 +2341,7 @@ std::optional StorageMergeTree::checkDataNext(DataValidationTasksPt { try { - checkDataPart(part, true); + checkDataPart(part, true, noop, /* is_cancelled */[]{ return false; }, /* throw_on_broken_projection */true); return CheckResult(part->name, true, ""); } catch (...) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 1702b52fa35..ee15a26f244 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8891,12 +8891,11 @@ IStorage::DataValidationTasksPtr StorageReplicatedMergeTree::getCheckTaskList( std::optional StorageReplicatedMergeTree::checkDataNext(DataValidationTasksPtr & check_task_list) { - if (auto part = assert_cast(check_task_list.get())->next()) { try { - return CheckResult(part_check_thread.checkPartAndFix(part->name)); + return part_check_thread.checkPartAndFix(part->name, /* recheck_after */nullptr, /* throw_on_broken_projection */true); } catch (const Exception & ex) { diff --git a/src/Storages/System/StorageSystemProjectionParts.cpp b/src/Storages/System/StorageSystemProjectionParts.cpp index 016705f4e66..b1494f2ba98 100644 --- a/src/Storages/System/StorageSystemProjectionParts.cpp +++ b/src/Storages/System/StorageSystemProjectionParts.cpp @@ -83,7 +83,11 @@ StorageSystemProjectionParts::StorageSystemProjectionParts(const StorageID & tab {"rows_where_ttl_info.expression", std::make_shared(std::make_shared())}, {"rows_where_ttl_info.min", std::make_shared(std::make_shared())}, - {"rows_where_ttl_info.max", std::make_shared(std::make_shared())} + {"rows_where_ttl_info.max", std::make_shared(std::make_shared())}, + + {"is_broken", std::make_shared()}, + {"exception_code", std::make_shared()}, + {"exception", std::make_shared()}, } ) { @@ -272,12 +276,38 @@ void StorageSystemProjectionParts::processNextStorage( add_ttl_info_map(part->ttl_infos.moves_ttl); if (columns_mask[src_index++]) - columns[res_index++]->insert(queryToString(part->default_codec->getCodecDesc())); + { + if (part->default_codec) + columns[res_index++]->insert(queryToString(part->default_codec->getCodecDesc())); + else + columns[res_index++]->insertDefault(); + } add_ttl_info_map(part->ttl_infos.recompression_ttl); add_ttl_info_map(part->ttl_infos.group_by_ttl); add_ttl_info_map(part->ttl_infos.rows_where_ttl); + { + if (columns_mask[src_index++]) + columns[res_index++]->insert(part->is_broken.load(std::memory_order_relaxed)); + + if (part->is_broken) + { + std::lock_guard lock(part->broken_reason_mutex); + if (columns_mask[src_index++]) + columns[res_index++]->insert(part->exception_code); + if (columns_mask[src_index++]) + columns[res_index++]->insert(part->exception); + } + else + { + if (columns_mask[src_index++]) + columns[res_index++]->insertDefault(); + if (columns_mask[src_index++]) + columns[res_index++]->insertDefault(); + } + } + /// _state column should be the latest. /// Do not use part->getState*, it can be changed from different thread if (has_state_column) diff --git a/tests/integration/test_broken_projections/__init__.py b/tests/integration/test_broken_projections/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_broken_projections/config.d/backups.xml b/tests/integration/test_broken_projections/config.d/backups.xml new file mode 100644 index 00000000000..4da8edffd67 --- /dev/null +++ b/tests/integration/test_broken_projections/config.d/backups.xml @@ -0,0 +1,13 @@ + + + + + local + /var/lib/clickhouse/disks/backups/ + + + + + backups + + diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py new file mode 100644 index 00000000000..4a4690a5d0a --- /dev/null +++ b/tests/integration/test_broken_projections/test.py @@ -0,0 +1,576 @@ +import time +import pytest +import logging +import string +import random +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node", + main_configs=["config.d/backups.xml"], + stay_alive=True, + with_zookeeper=True, + ) + + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + cluster.shutdown() + + +def create_table(node, table, replica, data_prefix="", aggressive_merge=True): + if data_prefix == "": + data_prefix = table + + if aggressive_merge: + vertical_merge_algorithm_min_rows_to_activate = 1 + vertical_merge_algorithm_min_columns_to_activate = 1 + max_parts_to_merge_at_once = 3 + else: + vertical_merge_algorithm_min_rows_to_activate = 100000 + vertical_merge_algorithm_min_columns_to_activate = 100 + max_parts_to_merge_at_once = 3 + + node.query( + f""" + DROP TABLE IF EXISTS {table} SYNC; + CREATE TABLE {table} + ( + a String, + b String, + c Int64, + d Int64, + e Int64, + PROJECTION proj1 + ( + SELECT c ORDER BY d + ), + PROJECTION proj2 + ( + SELECT d ORDER BY c + ) + ) + ENGINE = ReplicatedMergeTree('/test_broken_projection_{data_prefix}/data/', '{replica}') ORDER BY a + SETTINGS min_bytes_for_wide_part = 0, + max_parts_to_merge_at_once={max_parts_to_merge_at_once}, + enable_vertical_merge_algorithm=0, + vertical_merge_algorithm_min_rows_to_activate = {vertical_merge_algorithm_min_rows_to_activate}, + vertical_merge_algorithm_min_columns_to_activate = {vertical_merge_algorithm_min_columns_to_activate}, + compress_primary_key=0; + """ + ) + + +def insert(node, table, offset, size): + node.query( + f""" + INSERT INTO {table} + SELECT number, number, number, number, number%2 FROM numbers({offset}, {size}) + SETTINGS insert_keeper_fault_injection_probability=0.0; + """ + ) + + +def get_parts(node, table): + return ( + node.query( + f""" + SELECT name + FROM system.parts + WHERE table='{table}' AND database=currentDatabase() AND active = 1 + ORDER BY name;" + """ + ) + .strip() + .split("\n") + ) + + +def bash(node, command): + node.exec_in_container(["bash", "-c", command], privileged=True, user="root") + + +def break_projection(node, table, part, parent_part, break_type): + part_path = node.query( + f""" + SELECT path + FROM system.projection_parts + WHERE table='{table}' + AND database=currentDatabase() + AND active=1 + AND part_name='{part}' + AND parent_name='{parent_part}' + ORDER BY modification_time DESC + LIMIT 1; + """ + ).strip() + + node.query( + f"select throwIf(substring('{part_path}', 1, 1) != '/', 'Path is relative: {part_path}')" + ) + + if break_type == "data": + bash(node, f"rm '{part_path}/d.bin'") + bash(node, f"rm '{part_path}/c.bin'") + elif break_type == "metadata": + bash(node, f"rm '{part_path}/columns.txt'") + elif break_type == "part": + bash(node, f"rm -r '{part_path}'") + + +def break_part(node, table, part): + part_path = node.query( + f""" + SELECT path + FROM system.parts + WHERE table='{table}' + AND database=currentDatabase() + AND active=1 + AND part_name='{part}' + ORDER BY modification_time DESC + LIMIT 1; + """ + ).strip() + + node.query( + f"select throwIf(substring('{part_path}', 1, 1) != '/', 'Path is relative: {part_path}')" + ) + bash(node, f"rm '{part_path}/columns.txt'") + + +def get_broken_projections_info(node, table): + return node.query( + f""" + SELECT parent_name, name, errors.name FROM + ( + SELECT parent_name, name, exception_code + FROM system.projection_parts + WHERE table='{table}' + AND database=currentDatabase() + AND is_broken = 1 + ) AS parts_info + INNER JOIN system.errors AS errors + ON parts_info.exception_code = errors.code + ORDER BY parent_name, name + """ + ).strip() + + +def get_projections_info(node, table): + return node.query( + f""" + SELECT parent_name, name, is_broken + FROM system.projection_parts + WHERE table='{table}' + AND active = 1 + AND database=currentDatabase() + ORDER BY parent_name, name + """ + ).strip() + + +def optimize(node, table, final, no_wait): + query = f"OPTIMIZE TABLE {table}" + if final: + query += " FINAL" + if no_wait: + query += " SETTINGS alter_sync=0" + node.query(query) + + +def reattach(node, table): + node.query( + f""" + DETACH TABLE {table}; + ATTACH TABLE {table}; + """ + ) + + +def materialize_projection(node, table, proj): + node.query( + f"ALTER TABLE {table} MATERIALIZE PROJECTION {proj} SETTINGS mutations_sync=2" + ) + + +def check_table_full(node, table): + return node.query( + f"CHECK TABLE {table} SETTINGS check_query_single_value_result = 0;" + ).strip() + + +def random_str(length=6): + alphabet = string.ascii_lowercase + string.digits + return "".join(random.SystemRandom().choice(alphabet) for _ in range(length)) + + +def check(node, table, check_result, expect_broken_part="", expected_error=""): + if expect_broken_part == "proj1": + assert expected_error in node.query_and_get_error( + f"SELECT c FROM '{table}' WHERE d == 12 ORDER BY c" + ) + else: + query_id = node.query( + f"SELECT queryID() FROM (SELECT c FROM '{table}' WHERE d == 12 ORDER BY c)" + ).strip() + node.query("SYSTEM FLUSH LOGS") + res = node.query( + f""" + SELECT query, splitByChar('.', arrayJoin(projections))[-1] + FROM system.query_log + WHERE query_id='{query_id}' AND type='QueryFinish' + """ + ) + if res == "": + res = node.query( + """ + SELECT query_id, query, splitByChar('.', arrayJoin(projections))[-1] + FROM system.query_log ORDER BY query_start_time_microseconds DESC + """ + ) + print(f"LOG: {res}") + assert False + assert "proj1" in res + + if expect_broken_part == "proj2": + assert expected_error in node.query_and_get_error( + f"SELECT d FROM '{table}' WHERE c == 12 ORDER BY d" + ) + else: + query_id = node.query( + f"SELECT queryID() FROM (SELECT d FROM '{table}' WHERE c == 12 ORDER BY d)" + ).strip() + node.query("SYSTEM FLUSH LOGS") + res = node.query( + f""" + SELECT query, splitByChar('.', arrayJoin(projections))[-1] + FROM system.query_log + WHERE query_id='{query_id}' AND type='QueryFinish' + """ + ) + if res == "": + res = node.query( + """ + SELECT query_id, query, splitByChar('.', arrayJoin(projections))[-1] + FROM system.query_log ORDER BY query_start_time_microseconds DESC + """ + ) + print(f"LOG: {res}") + assert False + assert "proj2" in res + + assert check_result == int(node.query(f"CHECK TABLE {table}")) + + +def test_broken_ignored(cluster): + node = cluster.instances["node"] + + table_name = "test1" + create_table(node, table_name, 1) + + insert(node, table_name, 0, 5) + insert(node, table_name, 5, 5) + insert(node, table_name, 10, 5) + insert(node, table_name, 15, 5) + + assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_3_0"] == get_parts( + node, table_name + ) + + # Break metadata (columns.txt) file of projection 'proj1' + break_projection(node, table_name, "proj1", "all_2_2_0", "metadata") + + # Do select and after "check table" query. + # Select works because it does not read columns.txt. + # But expect check table result as 0. + check(node, table_name, 0) + + # Projection 'proj1' from part all_2_2_0 will now appear in broken parts info + # because it was marked broken during "check table" query. + assert "all_2_2_0\tproj1\tFILE_DOESNT_EXIST" in get_broken_projections_info( + node, table_name + ) + + # Check table query will also show a list of parts which have broken projections. + assert "all_2_2_0" in check_table_full(node, table_name) + + # Break data file of projection 'proj2' for part all_2_2_0 + break_projection(node, table_name, "proj2", "all_2_2_0", "data") + + # It will not yet appear in broken projections info. + assert "proj2" not in get_broken_projections_info(node, table_name) + + # Select now fails with error "File doesn't exist" + check(node, table_name, 0, "proj2", "FILE_DOESNT_EXIST") + + # Projection 'proj2' from part all_2_2_0 will now appear in broken parts info. + assert "all_2_2_0\tproj2\tNO_FILE_IN_DATA_PART" in get_broken_projections_info( + node, table_name + ) + + # Second select works, because projection is now marked as broken. + check(node, table_name, 0) + + # Break data file of projection 'proj2' for part all_3_3_0 + break_projection(node, table_name, "proj2", "all_3_3_0", "data") + + # It will not yet appear in broken projections info. + assert "all_3_3_0" not in get_broken_projections_info(node, table_name) + + insert(node, table_name, 20, 5) + insert(node, table_name, 25, 5) + + # Part all_3_3_0 has 'proj' and 'proj2' projections, but 'proj2' is broken and server does NOT know it yet. + # Parts all_4_4_0 and all_5_5_0 have both non-broken projections. + # So a merge will be create for future part all_3_5_1. + # During merge it will fail to read from 'proj2' of part all_3_3_0 and proj2 will be marked broken. + # Merge will be retried and on second attempt it will succeed. + # The result part all_3_5_1 will have only 1 projection - 'proj', because + # it will skip 'proj2' as it will see that one part does not have it anymore in the set of valid projections. + optimize(node, table_name, 0, 1) + time.sleep(5) + + # table_uuid=node.query(f"SELECT uuid FROM system.tables WHERE table='{table_name}' and database=currentDatabase()").strip() + # assert 0 < int( + # node.query( + # f""" + # SYSTEM FLUSH LOGS; + # SELECT count() FROM system.text_log + # WHERE level='Error' + # AND logger_name='MergeTreeBackgroundExecutor' + # AND message like 'Exception while executing background task %{table_uuid}:all_3_5_1%%Cannot open file%proj2.proj/c.bin%' + # """) + # ) + + assert "all_3_3_0" in get_broken_projections_info(node, table_name) + check(node, table_name, 0) + + +def test_materialize_broken_projection(cluster): + node = cluster.instances["node"] + + table_name = "test2" + create_table(node, table_name, 1) + + insert(node, table_name, 0, 5) + insert(node, table_name, 5, 5) + insert(node, table_name, 10, 5) + insert(node, table_name, 15, 5) + + assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_3_0"] == get_parts( + node, table_name + ) + + break_projection(node, table_name, "proj1", "all_1_1_0", "metadata") + reattach(node, table_name) + + assert "all_1_1_0\tproj1\tNO_FILE_IN_DATA_PART" in get_broken_projections_info( + node, table_name + ) + assert "Part all_1_1_0 has a broken projection proj1" in check_table_full( + node, table_name + ) + + break_projection(node, table_name, "proj2", "all_1_1_0", "data") + reattach(node, table_name) + + assert "all_1_1_0\tproj2\tFILE_DOESNT_EXIST" in get_broken_projections_info( + node, table_name + ) + assert "Part all_1_1_0 has a broken projection proj2" in check_table_full( + node, table_name + ) + + materialize_projection(node, table_name, "proj1") + + assert "has a broken projection" not in check_table_full(node, table_name) + + +def test_broken_ignored_replicated(cluster): + node = cluster.instances["node"] + + table_name = "test3" + table_name2 = "test3_replica" + create_table(node, table_name, 1) + + insert(node, table_name, 0, 5) + insert(node, table_name, 5, 5) + insert(node, table_name, 10, 5) + insert(node, table_name, 15, 5) + + check(node, table_name, 1) + + create_table(node, table_name2, 2, table_name) + check(node, table_name2, 1) + + break_projection(node, table_name, "proj1", "all_0_0_0", "data") + assert "Part all_0_0_0 has a broken projection proj1" in check_table_full( + node, table_name + ) + + break_part(node, table_name, "all_0_0_0") + node.query(f"SYSTEM SYNC REPLICA {table_name}") + assert "has a broken projection" not in check_table_full(node, table_name) + + +def get_random_string(string_length=8): + alphabet = string.ascii_letters + string.digits + return "".join((random.choice(alphabet) for _ in range(string_length))) + + +def test_broken_projections_in_backups_1(cluster): + node = cluster.instances["node"] + + table_name = "test4" + create_table(node, table_name, 1, aggressive_merge=False, data_prefix=table_name) + + node.query("SYSTEM STOP MERGES") + + insert(node, table_name, 0, 5) + insert(node, table_name, 5, 5) + insert(node, table_name, 10, 5) + insert(node, table_name, 15, 5) + + assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_3_0"] == get_parts( + node, table_name + ) + + check(node, table_name, 1) + + break_projection(node, table_name, "proj1", "all_2_2_0", "data") + check(node, table_name, 0, "proj1", "FILE_DOESNT_EXIST") + + assert "all_2_2_0\tproj1\tNO_FILE_IN_DATA_PART" in get_broken_projections_info( + node, table_name + ) + + backup_name = f"b1-{get_random_string()}" + assert "BACKUP_CREATED" in node.query( + f""" + set backup_restore_keeper_fault_injection_probability=0.0; + backup table {table_name} to Disk('backups', '{backup_name}') settings check_projection_parts=false; + """ + ) + + assert "RESTORED" in node.query( + f""" + drop table {table_name} sync; + set backup_restore_keeper_fault_injection_probability=0.0; + restore table {table_name} from Disk('backups', '{backup_name}'); + """ + ) + + node.query("SYSTEM STOP MERGES") + + check(node, table_name, 1) + assert "" == get_broken_projections_info(node, table_name) + + +def test_broken_projections_in_backups_2(cluster): + node = cluster.instances["node"] + + table_name = "test5" + create_table(node, table_name, 1, aggressive_merge=False, data_prefix=table_name) + + insert(node, table_name, 0, 5) + insert(node, table_name, 5, 5) + insert(node, table_name, 10, 5) + insert(node, table_name, 15, 5) + + assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_3_0"] == get_parts( + node, table_name + ) + + check(node, table_name, 1) + break_projection(node, table_name, "proj2", "all_2_2_0", "part") + check(node, table_name, 0, "proj2", "ErrnoException") + + assert "all_2_2_0\tproj2\tFILE_DOESNT_EXIST" == get_broken_projections_info( + node, table_name + ) + + assert "FILE_DOESNT_EXIST" in node.query_and_get_error( + f""" + set backup_restore_keeper_fault_injection_probability=0.0; + backup table {table_name} to Disk('backups', 'b2') + """ + ) + + materialize_projection(node, table_name, "proj2") + check(node, table_name, 1) + + backup_name = f"b3-{get_random_string()}" + assert "BACKUP_CREATED" in node.query( + f""" + set backup_restore_keeper_fault_injection_probability=0.0; + backup table {table_name} to Disk('backups', '{backup_name}') settings check_projection_parts=false; + """ + ) + + assert "RESTORED" in node.query( + f""" + drop table {table_name} sync; + set backup_restore_keeper_fault_injection_probability=0.0; + restore table {table_name} from Disk('backups', '{backup_name}'); + """ + ) + check(node, table_name, 1) + + +def test_broken_projections_in_backups_3(cluster): + node = cluster.instances["node"] + + table_name = "test6" + create_table(node, table_name, 1, aggressive_merge=False, data_prefix=table_name) + + node.query("SYSTEM STOP MERGES") + + insert(node, table_name, 0, 5) + insert(node, table_name, 5, 5) + insert(node, table_name, 10, 5) + insert(node, table_name, 15, 5) + + assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_3_0"] == get_parts( + node, table_name + ) + + check(node, table_name, 1) + + break_projection(node, table_name, "proj1", "all_1_1_0", "part") + assert "Part all_1_1_0 has a broken projection proj1" in check_table_full( + node, table_name + ) + assert "all_1_1_0\tproj1\tFILE_DOESNT_EXIST" == get_broken_projections_info( + node, table_name + ) + + backup_name = f"b4-{get_random_string()}" + assert "BACKUP_CREATED" in node.query( + f""" + set backup_restore_keeper_fault_injection_probability=0.0; + backup table {table_name} to Disk('backups', '{backup_name}') settings check_projection_parts=false, allow_backup_broken_projections=true; + """ + ) + + assert "RESTORED" in node.query( + f""" + drop table {table_name} sync; + set backup_restore_keeper_fault_injection_probability=0.0; + restore table {table_name} from Disk('backups', '{backup_name}'); + """ + ) + + check(node, table_name, 0) + assert "all_1_1_0\tproj1\tNO_FILE_IN_DATA_PART" == get_broken_projections_info( + node, table_name + ) diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 7382b24afbc..e60fb844de8 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -686,6 +686,9 @@ CREATE TABLE system.projection_parts `rows_where_ttl_info.expression` Array(String), `rows_where_ttl_info.min` Array(DateTime), `rows_where_ttl_info.max` Array(DateTime), + `is_broken` UInt8, + `exception_code` Int32, + `exception` String, `bytes` UInt64 ALIAS bytes_on_disk, `marks_size` UInt64 ALIAS marks_bytes, `part_name` String ALIAS name From 524a2ca72decc124ef1e38b79843c2388cceb0bb Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 27 Feb 2024 19:17:34 +0100 Subject: [PATCH 0042/1165] WIP on createForShard --- .../OptimizeShardingKeyRewriteInVisitor.cpp | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 3a592c0fe55..8aca28a90ef 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -5,6 +5,12 @@ #include #include #include +#include "Analyzer/ColumnNode.h" +#include "Analyzer/ConstantNode.h" +#include "Analyzer/FunctionNode.h" +#include "Analyzer/IQueryTreeNode.h" +#include "Analyzer/InDepthQueryTreeVisitor.h" +#include "DataTypes/IDataType.h" namespace { @@ -119,4 +125,42 @@ void OptimizeShardingKeyRewriteInMatcher::visit(ASTFunction & function, Data & d } } + +class OptimizeShardingKeyRewriteIn : InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + using Base::Base; + + void enterImpl(QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + if (!function_node || function_node->getFunctionName() != "in") + return; + + auto & arguments = function_node->getArguments().getNodes(); + auto * column = arguments[0]->as(); + if (!column) + return; + + if (!data.sharding_key_expr->getRequiredColumnsWithTypes().contains(column->getColumnName())) + return; + + if (auto * constant = arguments[1]->as()) + { + if (isTuple(constant->getResultType())) + { + auto & tuple = constant->getValue().get(); + std::erase_if(tuple, [&](auto & child) + { + return tuple.size() > 1 && !shardContains(child, name, data); + }); + } + } + } + + OptimizeShardingKeyRewriteInMatcher::Data data; +}; + + } From 0de2d766fa971f54eff40641e16ed6857e1ece5f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 28 Feb 2024 15:30:06 +0100 Subject: [PATCH 0043/1165] WIP on different JSONs on shards --- src/Analyzer/IdentifierNode.cpp | 10 +- src/Analyzer/IdentifierNode.h | 6 ++ src/DataTypes/ObjectUtils.cpp | 75 ++++++++++++- src/DataTypes/ObjectUtils.h | 11 ++ .../ClusterProxy/SelectStreamFactory.cpp | 79 +++++++++++++- .../ClusterProxy/SelectStreamFactory.h | 26 +++++ .../ClusterProxy/executeQuery.cpp | 101 ++++++++++++------ src/Interpreters/ClusterProxy/executeQuery.h | 2 - .../OptimizeShardingKeyRewriteInVisitor.cpp | 35 ++++-- .../OptimizeShardingKeyRewriteInVisitor.h | 3 + src/Processors/QueryPlan/ReadFromRemote.cpp | 10 +- src/Storages/StorageDistributed.cpp | 32 +++--- 12 files changed, 325 insertions(+), 65 deletions(-) diff --git a/src/Analyzer/IdentifierNode.cpp b/src/Analyzer/IdentifierNode.cpp index 88b3daacb12..7e4d4c02a4c 100644 --- a/src/Analyzer/IdentifierNode.cpp +++ b/src/Analyzer/IdentifierNode.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -56,13 +57,18 @@ void IdentifierNode::updateTreeHashImpl(HashState & state) const QueryTreeNodePtr IdentifierNode::cloneImpl() const { - return std::make_shared(identifier); + auto result = std::make_shared(identifier); + result->use_parts_for_to_ast = use_parts_for_to_ast; + return result; } ASTPtr IdentifierNode::toASTImpl(const ConvertToASTOptions & /* options */) const { auto identifier_parts = identifier.getParts(); - return std::make_shared(std::move(identifier_parts)); + if (use_parts_for_to_ast) + return std::make_shared(std::move(identifier_parts)); + else + return std::make_shared(identifier.getFullName()); } } diff --git a/src/Analyzer/IdentifierNode.h b/src/Analyzer/IdentifierNode.h index 872bb14d512..3bc37b4c69d 100644 --- a/src/Analyzer/IdentifierNode.h +++ b/src/Analyzer/IdentifierNode.h @@ -52,6 +52,11 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; + void useFullNameInToAST() + { + use_parts_for_to_ast = false; + } + protected: bool isEqualImpl(const IQueryTreeNode & rhs) const override; @@ -64,6 +69,7 @@ protected: private: Identifier identifier; std::optional table_expression_modifiers; + bool use_parts_for_to_ast = false; static constexpr size_t children_size = 0; }; diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 23d29136c85..01ba50d90f3 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -20,6 +21,16 @@ #include #include #include +#include "Analyzer/ConstantNode.h" +#include "Analyzer/FunctionNode.h" +#include "Analyzer/IQueryTreeNode.h" +#include "Analyzer/Identifier.h" +#include "Analyzer/IdentifierNode.h" +#include "Analyzer/QueryNode.h" +#include "Analyzer/Utils.h" +#include +#include +#include "Common/logger_useful.h" namespace DB @@ -888,10 +899,10 @@ static void addConstantToWithClause(const ASTPtr & query, const String & column_ /// @expected_columns and @available_columns contain descriptions /// of extended Object columns. -void replaceMissedSubcolumnsByConstants( +NamesAndTypes calculateMissedSubcolumns( const ColumnsDescription & expected_columns, - const ColumnsDescription & available_columns, - ASTPtr query) + const ColumnsDescription & available_columns +) { NamesAndTypes missed_names_types; @@ -928,6 +939,18 @@ void replaceMissedSubcolumnsByConstants( [](const auto & lhs, const auto & rhs) { return lhs.name < rhs.name; }); } + return missed_names_types; +} + +/// @expected_columns and @available_columns contain descriptions +/// of extended Object columns. +void replaceMissedSubcolumnsByConstants( + const ColumnsDescription & expected_columns, + const ColumnsDescription & available_columns, + ASTPtr query) +{ + NamesAndTypes missed_names_types = calculateMissedSubcolumns(expected_columns, available_columns); + if (missed_names_types.empty()) return; @@ -940,6 +963,52 @@ void replaceMissedSubcolumnsByConstants( addConstantToWithClause(query, name, type); } +/// @expected_columns and @available_columns contain descriptions +/// of extended Object columns. +void replaceMissedSubcolumnsByConstants( + const ColumnsDescription & expected_columns, + const ColumnsDescription & available_columns, + QueryTreeNodePtr & query, + const ContextPtr & context [[maybe_unused]]) +{ + NamesAndTypes missed_names_types = calculateMissedSubcolumns(expected_columns, available_columns); + + if (missed_names_types.empty()) + return; + + auto * query_node = query->as(); + if (!query_node) + return; + + auto table_expression = extractLeftTableExpression(query_node->getJoinTree()); + + auto & with_nodes = query_node->getWith().getNodes(); + + std::unordered_map column_name_to_node; + for (const auto & [name, type] : missed_names_types) + { + auto constant = std::make_shared(type->getDefault(), type); + constant->setAlias(table_expression->getAlias() + name); + // auto materialize = std::make_shared("materialize"); + + // auto function = FunctionFactory::instance().get("materialize", context); + // materialize->getArguments().getNodes() = { constant }; + // materialize->resolveAsFunction(function->build(materialize->getArgumentColumns())); + // materialize->setAlias(name); + + with_nodes.push_back(constant); + + auto id = std::make_shared(Identifier(table_expression->getAlias() + name)); + id->useFullNameInToAST(); + column_name_to_node[name] = id; + LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Name {} Expression\n{}", name, column_name_to_node[name]->dumpTree()); + } + + LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Table expression\n{} ", table_expression->dumpTree()); + replaceColumns(query, table_expression, column_name_to_node); + LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Result:\n{} ", query->dumpTree()); +} + Field FieldVisitorReplaceScalars::operator()(const Array & x) const { if (num_dimensions_to_keep == 0) diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 2bfcaae09ca..f4a8abe8abf 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -3,6 +3,8 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" +#include "Interpreters/Context_fwd.h" #include #include #include @@ -14,6 +16,9 @@ namespace DB struct StorageSnapshot; using StorageSnapshotPtr = std::shared_ptr; +class IQueryTreeNode; +using QueryTreeNodePtr = std::shared_ptr; + /// Returns number of dimensions in Array type. 0 if type is not array. size_t getNumberOfDimensions(const IDataType & type); @@ -97,6 +102,12 @@ void replaceMissedSubcolumnsByConstants( const ColumnsDescription & available_columns, ASTPtr query); +void replaceMissedSubcolumnsByConstants( + const ColumnsDescription & expected_columns, + const ColumnsDescription & available_columns, + QueryTreeNodePtr & query, + const ContextPtr & context); + /// Visitor that keeps @num_dimensions_to_keep dimensions in arrays /// and replaces all scalars or nested arrays to @replacement at that level. class FieldVisitorReplaceScalars : public StaticVisitor diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index f0592735caf..5167ffc0e27 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -5,6 +5,10 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" +#include "Interpreters/InterpreterSelectQueryAnalyzer.h" +#include "Interpreters/SelectQueryOptions.h" +#include "Planner/Utils.h" #include #include #include @@ -124,18 +128,55 @@ void SelectStreamFactory::createForShard( if (it != objects_by_shard.end()) replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, query_ast); + createForShardImpl( + shard_info, + query_ast, + main_table, + table_func_ptr, + std::move(context), + local_plans, + remote_shards, + shard_count, + parallel_replicas_enabled, + std::move(shard_filter_generator)); +} + +void SelectStreamFactory::createForShardImpl( + const Cluster::ShardInfo & shard_info, + const ASTPtr & query_ast, + const StorageID & main_table, + const ASTPtr & table_func_ptr, + ContextPtr context, + std::vector & local_plans, + Shards & remote_shards, + UInt32 shard_count, + bool parallel_replicas_enabled, + AdditionalShardFilterGenerator shard_filter_generator) +{ auto emplace_local_stream = [&]() { + Block shard_header; + if (context->getSettingsRef().allow_experimental_analyzer) + shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_ast, context, SelectQueryOptions(processed_stage).analyze()); + else + shard_header = header; + local_plans.emplace_back(createLocalPlan( - query_ast, header, context, processed_stage, shard_info.shard_num, shard_count)); + query_ast, shard_header, context, processed_stage, shard_info.shard_num, shard_count)); }; auto emplace_remote_stream = [&](bool lazy = false, time_t local_delay = 0) { + Block shard_header; + if (context->getSettingsRef().allow_experimental_analyzer) + shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_ast, context, SelectQueryOptions(processed_stage).analyze()); + else + shard_header = header; + remote_shards.emplace_back(Shard{ .query = query_ast, .main_table = main_table, - .header = header, + .header = shard_header, .shard_info = shard_info, .lazy = lazy, .local_delay = local_delay, @@ -243,6 +284,40 @@ void SelectStreamFactory::createForShard( emplace_remote_stream(); } +void SelectStreamFactory::createForShard( + const Cluster::ShardInfo & shard_info, + const QueryTreeNodePtr & query_tree, + const StorageID & main_table, + const ASTPtr & table_func_ptr, + ContextPtr context, + std::vector & local_plans, + Shards & remote_shards, + UInt32 shard_count, + bool parallel_replicas_enabled, + AdditionalShardFilterGenerator shard_filter_generator) +{ + + auto it = objects_by_shard.find(shard_info.shard_num); + QueryTreeNodePtr modified_query = query_tree; + if (it != objects_by_shard.end()) + replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); + + auto query_ast = queryNodeToDistributedSelectQuery(modified_query); + + createForShardImpl( + shard_info, + query_ast, + main_table, + table_func_ptr, + std::move(context), + local_plans, + remote_shards, + shard_count, + parallel_replicas_enabled, + std::move(shard_filter_generator)); + +} + } } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 9993ea7028d..45d6ea14c01 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -7,6 +7,7 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" namespace DB { @@ -83,10 +84,35 @@ public: bool parallel_replicas_enabled, AdditionalShardFilterGenerator shard_filter_generator); + void createForShard( + const Cluster::ShardInfo & shard_info, + const QueryTreeNodePtr & query_tree, + const StorageID & main_table, + const ASTPtr & table_func_ptr, + ContextPtr context, + std::vector & local_plans, + Shards & remote_shards, + UInt32 shard_count, + bool parallel_replicas_enabled, + AdditionalShardFilterGenerator shard_filter_generator); + const Block header; const ColumnsDescriptionByShardNum objects_by_shard; const StorageSnapshotPtr storage_snapshot; QueryProcessingStage::Enum processed_stage; + +private: + void createForShardImpl( + const Cluster::ShardInfo & shard_info, + const ASTPtr & query_ast, + const StorageID & main_table, + const ASTPtr & table_func_ptr, + ContextPtr context, + std::vector & local_plans, + Shards & remote_shards, + UInt32 shard_count, + bool parallel_replicas_enabled, + AdditionalShardFilterGenerator shard_filter_generator); }; } diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 6cdff939af1..07ef7aa6c96 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -204,12 +204,10 @@ void executeQuery( const ASTPtr & table_func_ptr, SelectStreamFactory & stream_factory, LoggerPtr log, - const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster, const DistributedSettings & distributed_settings, AdditionalShardFilterGenerator shard_filter_generator) { @@ -218,6 +216,8 @@ void executeQuery( if (settings.max_distributed_depth && context->getClientInfo().distributed_depth >= settings.max_distributed_depth) throw Exception(ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH, "Maximum distributed depth exceeded"); + const ClusterPtr & not_optimized_cluster = query_info.cluster; + std::vector plans; SelectStreamFactory::Shards remote_shards; @@ -237,40 +237,81 @@ void executeQuery( new_context->increaseDistributedDepth(); const size_t shards = cluster->getShardCount(); - for (size_t i = 0, s = cluster->getShardsInfo().size(); i < s; ++i) + + if (context->getSettingsRef().allow_experimental_analyzer) { - const auto & shard_info = cluster->getShardsInfo()[i]; - - ASTPtr query_ast_for_shard = query_ast->clone(); - if (sharding_key_expr && query_info.optimized_cluster && settings.optimize_skip_unused_shards_rewrite_in && shards > 1) + for (size_t i = 0, s = cluster->getShardsInfo().size(); i < s; ++i) { - OptimizeShardingKeyRewriteInVisitor::Data visitor_data{ - sharding_key_expr, - sharding_key_expr->getSampleBlock().getByPosition(0).type, - sharding_key_column_name, + const auto & shard_info = cluster->getShardsInfo()[i]; + + auto query_for_shard = query_info.query_tree->clone(); + if (sharding_key_expr && query_info.optimized_cluster && settings.optimize_skip_unused_shards_rewrite_in && shards > 1) + { + OptimizeShardingKeyRewriteInVisitor::Data visitor_data{ + sharding_key_expr, + sharding_key_expr->getSampleBlock().getByPosition(0).type, + sharding_key_column_name, + shard_info, + not_optimized_cluster->getSlotToShard(), + }; + optimizeShardingKeyRewriteIn(query_for_shard, std::move(visitor_data), new_context); + } + + // decide for each shard if parallel reading from replicas should be enabled + // according to settings and number of replicas declared per shard + const auto & addresses = cluster->getShardsAddresses().at(i); + bool parallel_replicas_enabled = addresses.size() > 1 && context->canUseTaskBasedParallelReplicas(); + + stream_factory.createForShard( shard_info, - not_optimized_cluster->getSlotToShard(), - }; - OptimizeShardingKeyRewriteInVisitor visitor(visitor_data); - visitor.visit(query_ast_for_shard); + query_for_shard, + main_table, + table_func_ptr, + new_context, + plans, + remote_shards, + static_cast(shards), + parallel_replicas_enabled, + shard_filter_generator); } + } + else + { + for (size_t i = 0, s = cluster->getShardsInfo().size(); i < s; ++i) + { + const auto & shard_info = cluster->getShardsInfo()[i]; - // decide for each shard if parallel reading from replicas should be enabled - // according to settings and number of replicas declared per shard - const auto & addresses = cluster->getShardsAddresses().at(i); - bool parallel_replicas_enabled = addresses.size() > 1 && context->canUseTaskBasedParallelReplicas(); + ASTPtr query_ast_for_shard = query_info.query->clone(); + if (sharding_key_expr && query_info.optimized_cluster && settings.optimize_skip_unused_shards_rewrite_in && shards > 1) + { + OptimizeShardingKeyRewriteInVisitor::Data visitor_data{ + sharding_key_expr, + sharding_key_expr->getSampleBlock().getByPosition(0).type, + sharding_key_column_name, + shard_info, + not_optimized_cluster->getSlotToShard(), + }; + OptimizeShardingKeyRewriteInVisitor visitor(visitor_data); + visitor.visit(query_ast_for_shard); + } - stream_factory.createForShard( - shard_info, - query_ast_for_shard, - main_table, - table_func_ptr, - new_context, - plans, - remote_shards, - static_cast(shards), - parallel_replicas_enabled, - shard_filter_generator); + // decide for each shard if parallel reading from replicas should be enabled + // according to settings and number of replicas declared per shard + const auto & addresses = cluster->getShardsAddresses().at(i); + bool parallel_replicas_enabled = addresses.size() > 1 && context->canUseTaskBasedParallelReplicas(); + + stream_factory.createForShard( + shard_info, + query_ast_for_shard, + main_table, + table_func_ptr, + new_context, + plans, + remote_shards, + static_cast(shards), + parallel_replicas_enabled, + shard_filter_generator); + } } if (!remote_shards.empty()) diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index bbc3c6c9e49..8f6f6300c7b 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -58,12 +58,10 @@ void executeQuery( const ASTPtr & table_func_ptr, SelectStreamFactory & stream_factory, LoggerPtr log, - const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster, const DistributedSettings & distributed_settings, AdditionalShardFilterGenerator shard_filter_generator); diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 8aca28a90ef..42c6e63da01 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -11,6 +12,7 @@ #include "Analyzer/IQueryTreeNode.h" #include "Analyzer/InDepthQueryTreeVisitor.h" #include "DataTypes/IDataType.h" +#include "Interpreters/Context_fwd.h" namespace { @@ -126,11 +128,15 @@ void OptimizeShardingKeyRewriteInMatcher::visit(ASTFunction & function, Data & d } -class OptimizeShardingKeyRewriteIn : InDepthQueryTreeVisitorWithContext +class OptimizeShardingKeyRewriteIn : public InDepthQueryTreeVisitorWithContext { public: using Base = InDepthQueryTreeVisitorWithContext; - using Base::Base; + + OptimizeShardingKeyRewriteIn(OptimizeShardingKeyRewriteInVisitor::Data data_, ContextPtr context) + : Base(std::move(context)) + , data(std::move(data_)) + {} void enterImpl(QueryTreeNodePtr & node) { @@ -143,6 +149,8 @@ public: if (!column) return; + auto name = column->getColumnName(); + if (!data.sharding_key_expr->getRequiredColumnsWithTypes().contains(column->getColumnName())) return; @@ -150,17 +158,30 @@ public: { if (isTuple(constant->getResultType())) { - auto & tuple = constant->getValue().get(); - std::erase_if(tuple, [&](auto & child) + const auto & tuple = constant->getValue().get(); + Tuple new_tuple; + new_tuple.reserve(tuple.size()); + + for (const auto & child : tuple) { - return tuple.size() > 1 && !shardContains(child, name, data); - }); + if (shardContains(child, name, data)) + new_tuple.push_back(child); + } + + if (new_tuple.empty()) + new_tuple.push_back(tuple.back()); + node = std::make_shared(new_tuple); } } } - OptimizeShardingKeyRewriteInMatcher::Data data; + OptimizeShardingKeyRewriteInVisitor::Data data; }; +void optimizeShardingKeyRewriteIn(QueryTreeNodePtr & node, OptimizeShardingKeyRewriteInVisitor::Data data, ContextPtr context) +{ + OptimizeShardingKeyRewriteIn visitor(std::move(data), std::move(context)); + visitor.visit(node); +} } diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h index d546db40df7..d202609160b 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h @@ -2,6 +2,7 @@ #include #include +#include "Analyzer/IQueryTreeNode.h" namespace DB { @@ -44,4 +45,6 @@ struct OptimizeShardingKeyRewriteInMatcher using OptimizeShardingKeyRewriteInVisitor = InDepthNodeVisitor; +void optimizeShardingKeyRewriteIn(QueryTreeNodePtr & node, OptimizeShardingKeyRewriteInVisitor::Data data, ContextPtr context); + } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 93c73a66b78..022c4f699f2 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -216,7 +216,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), shard.header); } void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard) @@ -281,7 +281,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact auto remote_query_executor = std::make_shared( shard.shard_info.pool, query_string, - output_stream->header, + shard.header, context, throttler, scalars, @@ -297,7 +297,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), shard.header); } } else @@ -305,7 +305,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact const String query_string = formattedAST(shard.query); auto remote_query_executor = std::make_shared( - shard.shard_info.pool, query_string, output_stream->header, context, throttler, scalars, external_tables, stage); + shard.shard_info.pool, query_string, shard.header, context, throttler, scalars, external_tables, stage); remote_query_executor->setLogger(log); if (context->canUseTaskBasedParallelReplicas()) @@ -326,7 +326,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), shard.header); } } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 92e7dcdf4f2..34ab21a4751 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -30,6 +30,7 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" #include #include @@ -813,7 +814,8 @@ void StorageDistributed::read( const size_t /*num_streams*/) { Block header; - ASTPtr query_ast; + + SelectQueryInfo modified_query_info = query_info; if (local_context->getSettingsRef().allow_experimental_analyzer) { @@ -821,7 +823,7 @@ void StorageDistributed::read( if (!remote_table_function_ptr) remote_storage_id = StorageID{remote_database, remote_table}; - auto query_tree_distributed = buildQueryTreeDistributed(query_info, + auto query_tree_distributed = buildQueryTreeDistributed(modified_query_info, storage_snapshot, remote_storage_id, remote_table_function_ptr); @@ -831,20 +833,24 @@ void StorageDistributed::read( */ for (auto & column : header) column.column = column.column->convertToFullColumnIfConst(); - query_ast = queryNodeToDistributedSelectQuery(query_tree_distributed); + modified_query_info.query = queryNodeToDistributedSelectQuery(query_tree_distributed); + + modified_query_info.query_tree = std::move(query_tree_distributed); } else { - header = InterpreterSelectQuery(query_info.query, local_context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); - query_ast = query_info.query; + header = InterpreterSelectQuery(modified_query_info.query, local_context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); } - const auto & modified_query_ast = ClusterProxy::rewriteSelectQuery( - local_context, query_ast, - remote_database, remote_table, remote_table_function_ptr); + if (!local_context->getSettingsRef().allow_experimental_analyzer) + { + modified_query_info.query = ClusterProxy::rewriteSelectQuery( + local_context, modified_query_info.query, + remote_database, remote_table, remote_table_function_ptr); + } /// Return directly (with correct header) if no shard to query. - if (query_info.getCluster()->getShardsInfo().empty()) + if (modified_query_info.getCluster()->getShardsInfo().empty()) { if (local_context->getSettingsRef().allow_experimental_analyzer) return; @@ -872,7 +878,7 @@ void StorageDistributed::read( const auto & settings = local_context->getSettingsRef(); ClusterProxy::AdditionalShardFilterGenerator additional_shard_filter_generator; - if (local_context->canUseParallelReplicasCustomKey(*query_info.getCluster())) + if (local_context->canUseParallelReplicasCustomKey(*modified_query_info.getCluster())) { if (auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *local_context)) { @@ -881,7 +887,7 @@ void StorageDistributed::read( column_description = this->getInMemoryMetadataPtr()->columns, custom_key_type = settings.parallel_replicas_custom_key_filter_type.value, context = local_context, - replica_count = query_info.getCluster()->getShardsInfo().front().per_replica_pools.size()](uint64_t replica_num) -> ASTPtr + replica_count = modified_query_info.getCluster()->getShardsInfo().front().per_replica_pools.size()](uint64_t replica_num) -> ASTPtr { return getCustomKeyFilterForParallelReplica( replica_count, replica_num - 1, my_custom_key_ast, custom_key_type, column_description, context); @@ -897,12 +903,10 @@ void StorageDistributed::read( remote_table_function_ptr, select_stream_factory, log, - modified_query_ast, local_context, - query_info, + modified_query_info, sharding_key_expr, sharding_key_column_name, - query_info.cluster, distributed_settings, additional_shard_filter_generator); From 8bf7c2c5971afc22dda32f9f4ad453ac481f2359 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 28 Feb 2024 15:40:42 +0100 Subject: [PATCH 0044/1165] Use output header --- src/Processors/QueryPlan/ReadFromRemote.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 022c4f699f2..fde2313bc15 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -216,7 +216,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); - addConvertingActions(pipes.back(), shard.header); + addConvertingActions(pipes.back(), output_stream->header); } void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard) @@ -297,7 +297,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), shard.header); + addConvertingActions(pipes.back(), output_stream->header); } } else @@ -326,7 +326,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), shard.header); + addConvertingActions(pipes.back(), output_stream->header); } } From 974ba7364f193838f735a9233c6dec4298172542 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Thu, 29 Feb 2024 00:55:17 +0100 Subject: [PATCH 0045/1165] better --- src/Disks/ObjectStorages/S3/diskSettings.cpp | 16 +++++++++++++--- src/IO/S3/Client.cpp | 2 ++ src/IO/S3/URI.cpp | 2 +- src/IO/WriteBufferFromS3.h | 2 +- src/Storages/StorageS3.cpp | 4 ++++ 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index b8688cd3de6..10172805f06 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -1,5 +1,6 @@ #include -#include "IO/S3/Client.h" +#include +#include #if USE_AWS_S3 @@ -10,7 +11,7 @@ #include #include #include -#include "Disks/DiskFactory.h" +#include #include #include @@ -25,6 +26,11 @@ namespace DB { +namespace ErrorCodes +{ +extern const int NO_ELEMENTS_IN_CONFIG; +} + std::unique_ptr getSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context) { const Settings & settings = context->getSettingsRef(); @@ -47,11 +53,15 @@ std::unique_ptr getClient( const Settings & global_settings = context->getGlobalContext()->getSettingsRef(); const Settings & local_settings = context->getSettingsRef(); - String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); + const String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); if (!uri.key.ends_with('/')) uri.key.push_back('/'); + if (S3::isS3ExpressEndpoint(endpoint) && !config.has(config_prefix + ".region")) + throw Exception( + ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Region should be explicitly specified for directory buckets ({})", config_prefix); + S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( config.getString(config_prefix + ".region", ""), context->getRemoteHostFilter(), diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index a75d41df3d1..4f93aba2f84 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -1,4 +1,5 @@ #include +#include #if USE_AWS_S3 @@ -965,6 +966,7 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT bool isS3ExpressEndpoint(const std::string & endpoint) { + /// On one hand this check isn't 100% reliable, on the other - all it will change is whether we attach checksums to the requests. return endpoint.contains("s3express"); } } diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 062d3b80850..027cb624ed5 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -122,7 +122,7 @@ URI::URI(const std::string & uri_) "Object storage system name is unrecognized in virtual hosted style S3 URI: {}", quoteString(name)); - if (name == COS || name == COSN) + if (name == COS) storage_name = COSN; else storage_name = name; diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 148cd27f854..59f4e19e15b 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -100,7 +100,7 @@ private: /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts. String multipart_upload_id; std::deque multipart_tags; - std::deque multipart_checksums; + std::deque multipart_checksums; // if enabled bool multipart_upload_finished = false; /// Track that prefinalize() is called only once diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 044a1ca5362..f96ff8b7eb6 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -133,6 +133,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int CANNOT_COMPILE_REGEXP; extern const int FILE_DOESNT_EXIST; + extern const int NO_ELEMENTS_IN_CONFIG; } @@ -1403,6 +1404,9 @@ void StorageS3::Configuration::connect(const ContextPtr & context) const Settings & global_settings = context->getGlobalContext()->getSettingsRef(); const Settings & local_settings = context->getSettingsRef(); + if (S3::isS3ExpressEndpoint(url.endpoint) && auth_settings.region.empty()) + throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Region should be explicitly specified for directory buckets"); + S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( auth_settings.region, context->getRemoteHostFilter(), From 37917a3ed34df22756562a04a90d3c985ca23bd8 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Thu, 29 Feb 2024 01:42:32 +0100 Subject: [PATCH 0046/1165] better --- src/IO/S3/Requests.h | 12 ++++++++++++ src/IO/WriteBufferFromS3.cpp | 10 +++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index 6f82a0f39d3..196f074c9df 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -39,6 +40,17 @@ inline void setPartChecksum(Model::CompletedPart & part, const std::string & che part.SetChecksumCRC32(checksum); } +inline void setRequestChecksum(Model::UploadPartRequest & req, const std::string & checksum) +{ + req.SetChecksumCRC32(checksum); +} + +inline std::string calculateChecksum(Model::UploadPartRequest & req) +{ + chassert(req.GetChecksumAlgorithm() == Aws::S3::Model::ChecksumAlgorithm::CRC32); + return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateCRC32(*(req.GetBody()))); +} + template inline void setChecksumAlgorithm(R & request) { diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index a162992278f..80ca96b0382 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -18,10 +18,6 @@ #include #include -#include -#include -#include - #include @@ -461,9 +457,9 @@ S3::UploadPartRequest WriteBufferFromS3::getUploadRequest(size_t part_number, Pa /// Checksums need to be provided on CompleteMultipartUpload requests, so we calculate then manually and store in multipart_checksums if (client_ptr->isS3ExpressBucket()) { - chassert(req.GetChecksumAlgorithm() == Aws::S3::Model::ChecksumAlgorithm::CRC32); - req.SetChecksumCRC32(Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateCRC32(*(req.GetBody())))); - multipart_checksums.push_back(req.GetChecksumCRC32()); + auto checksum = S3::RequestChecksum::calculateChecksum(req); + S3::RequestChecksum::setRequestChecksum(req, checksum); + multipart_checksums.push_back(std::move(checksum)); } return req; From 8b1a1d42daa01e946aa8102d683dbab90b447838 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Thu, 29 Feb 2024 18:07:00 +0300 Subject: [PATCH 0047/1165] Traverse shadow directory for system.remote_data_paths --- src/Core/Settings.h | 1 + src/Disks/IDisk.h | 5 ++++- src/Disks/ObjectStorages/DiskObjectStorage.cpp | 10 ++++++++-- src/Disks/ObjectStorages/DiskObjectStorage.h | 5 ++++- .../System/StorageSystemRemoteDataPaths.cpp | 13 +++++++++++++ 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d77b3a45188..7cf068d7f1f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -843,6 +843,7 @@ class IColumn; M(Bool, optimize_uniq_to_count, true, "Rewrite uniq and its variants(except uniqUpTo) to count if subquery has distinct or group by clause.", 0) \ M(Bool, use_variant_as_common_type, false, "Use Variant as a result type for if/multiIf in case when there is no common type for arguments", 0) \ M(Bool, enable_order_by_all, true, "Enable sorting expression ORDER BY ALL.", 0) \ + M(Bool, traverse_shadow_remote_data_paths, false, "Traverse shadow directory when query system.remote_data_paths", 0) \ \ /** Experimental functions */ \ M(Bool, allow_experimental_materialized_postgresql_table, false, "Allows to use the MaterializedPostgreSQL table engine. Disabled by default, because this feature is experimental", 0) \ diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 3d228850537..62b02938d1a 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -320,7 +320,10 @@ public: {} }; - virtual void getRemotePathsRecursive(const String &, std::vector &) + virtual void getRemotePathsRecursive( + const String &, + std::vector &, + const std::function & /* skip_predicate */ = {}) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getRemotePathsRecursive() not implemented for disk: {}`", diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 2a648f28f14..460d242d5cd 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -90,11 +90,17 @@ StoredObjects DiskObjectStorage::getStorageObjects(const String & local_path) co return metadata_storage->getStorageObjects(local_path); } -void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std::vector & paths_map) +void DiskObjectStorage::getRemotePathsRecursive( + const String & local_path, + std::vector & paths_map, + const std::function & skip_predicate) { if (!metadata_storage->exists(local_path)) return; + if (skip_predicate && skip_predicate(local_path)) + return; + /// Protect against concurrent delition of files (for example because of a merge). if (metadata_storage->isFile(local_path)) { @@ -142,7 +148,7 @@ void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std:: } for (; it->isValid(); it->next()) - DiskObjectStorage::getRemotePathsRecursive(fs::path(local_path) / it->name(), paths_map); + DiskObjectStorage::getRemotePathsRecursive(fs::path(local_path) / it->name(), paths_map, skip_predicate); } } diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index e1576509713..d7af656bea3 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -48,7 +48,10 @@ public: StoredObjects getStorageObjects(const String & local_path) const override; - void getRemotePathsRecursive(const String & local_path, std::vector & paths_map) override; + void getRemotePathsRecursive( + const String & local_path, + std::vector & paths_map, + const std::function & skip_predicate = {}) override; const std::string & getCacheName() const override { return object_storage->getCacheName(); } diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index 87b7a84e8ba..708c1369965 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -9,6 +9,7 @@ #include #include +namespace fs = std::filesystem; namespace DB { @@ -59,6 +60,18 @@ Pipe StorageSystemRemoteDataPaths::read( std::vector remote_paths_by_local_path; disk->getRemotePathsRecursive("store", remote_paths_by_local_path); disk->getRemotePathsRecursive("data", remote_paths_by_local_path); + if (context->getSettingsRef().traverse_shadow_remote_data_paths) + disk->getRemotePathsRecursive( + "shadow", + remote_paths_by_local_path, + [](const String & local_path) + { + // `shadow/{backup_name}/revision.txt` is not an object metadata file + const auto path = fs::path(local_path); + return path.filename() == "revision.txt" && + path.parent_path().has_parent_path() && + path.parent_path().parent_path().filename() == "shadow"; + }); FileCachePtr cache; From 5641fd8ba9c4f27794367e22632365df5cdf0303 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 29 Feb 2024 16:13:05 +0100 Subject: [PATCH 0048/1165] Fix build after merge --- src/IO/WriteBufferFromS3TaskTracker.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/WriteBufferFromS3TaskTracker.h b/src/IO/WriteBufferFromS3TaskTracker.h index 134abbbc4c1..4061f084a76 100644 --- a/src/IO/WriteBufferFromS3TaskTracker.h +++ b/src/IO/WriteBufferFromS3TaskTracker.h @@ -1,7 +1,7 @@ #pragma once #include "config.h" -#include +#include #include "WriteBufferFromS3.h" #include From 55053dae4459b1d1a6c05d436d1ab421a96c3934 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 29 Feb 2024 19:18:06 +0100 Subject: [PATCH 0049/1165] Some progress --- src/Analyzer/IdentifierNode.cpp | 10 +--- src/Analyzer/IdentifierNode.h | 6 --- src/DataTypes/ObjectUtils.cpp | 24 +++++----- src/DataTypes/ObjectUtils.h | 4 +- .../ClusterProxy/SelectStreamFactory.cpp | 10 ++-- .../ClusterProxy/SelectStreamFactory.h | 8 +++- src/Processors/QueryPlan/ReadFromRemote.cpp | 47 +++++++++++++++++++ .../test_distributed_type_object/test.py | 2 +- 8 files changed, 80 insertions(+), 31 deletions(-) diff --git a/src/Analyzer/IdentifierNode.cpp b/src/Analyzer/IdentifierNode.cpp index 7e4d4c02a4c..88b3daacb12 100644 --- a/src/Analyzer/IdentifierNode.cpp +++ b/src/Analyzer/IdentifierNode.cpp @@ -1,4 +1,3 @@ -#include #include #include @@ -57,18 +56,13 @@ void IdentifierNode::updateTreeHashImpl(HashState & state) const QueryTreeNodePtr IdentifierNode::cloneImpl() const { - auto result = std::make_shared(identifier); - result->use_parts_for_to_ast = use_parts_for_to_ast; - return result; + return std::make_shared(identifier); } ASTPtr IdentifierNode::toASTImpl(const ConvertToASTOptions & /* options */) const { auto identifier_parts = identifier.getParts(); - if (use_parts_for_to_ast) - return std::make_shared(std::move(identifier_parts)); - else - return std::make_shared(identifier.getFullName()); + return std::make_shared(std::move(identifier_parts)); } } diff --git a/src/Analyzer/IdentifierNode.h b/src/Analyzer/IdentifierNode.h index 3bc37b4c69d..872bb14d512 100644 --- a/src/Analyzer/IdentifierNode.h +++ b/src/Analyzer/IdentifierNode.h @@ -52,11 +52,6 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; - void useFullNameInToAST() - { - use_parts_for_to_ast = false; - } - protected: bool isEqualImpl(const IQueryTreeNode & rhs) const override; @@ -69,7 +64,6 @@ protected: private: Identifier identifier; std::optional table_expression_modifiers; - bool use_parts_for_to_ast = false; static constexpr size_t children_size = 0; }; diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 01ba50d90f3..47d8c5c9113 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -965,30 +965,32 @@ void replaceMissedSubcolumnsByConstants( /// @expected_columns and @available_columns contain descriptions /// of extended Object columns. -void replaceMissedSubcolumnsByConstants( +MissingObjectList replaceMissedSubcolumnsByConstants( const ColumnsDescription & expected_columns, const ColumnsDescription & available_columns, QueryTreeNodePtr & query, const ContextPtr & context [[maybe_unused]]) { + MissingObjectList missed_list; + NamesAndTypes missed_names_types = calculateMissedSubcolumns(expected_columns, available_columns); if (missed_names_types.empty()) - return; + return missed_list; auto * query_node = query->as(); if (!query_node) - return; + return missed_list; + + missed_list.reserve(missed_names_types.size()); auto table_expression = extractLeftTableExpression(query_node->getJoinTree()); - auto & with_nodes = query_node->getWith().getNodes(); - std::unordered_map column_name_to_node; for (const auto & [name, type] : missed_names_types) { auto constant = std::make_shared(type->getDefault(), type); - constant->setAlias(table_expression->getAlias() + name); + constant->setAlias(table_expression->getAlias() + "." + name); // auto materialize = std::make_shared("materialize"); // auto function = FunctionFactory::instance().get("materialize", context); @@ -996,17 +998,17 @@ void replaceMissedSubcolumnsByConstants( // materialize->resolveAsFunction(function->build(materialize->getArgumentColumns())); // materialize->setAlias(name); - with_nodes.push_back(constant); - - auto id = std::make_shared(Identifier(table_expression->getAlias() + name)); - id->useFullNameInToAST(); - column_name_to_node[name] = id; + column_name_to_node[name] = buildCastFunction(constant, type, context); + missed_list.push_back({ constant->getValueStringRepresentation() + "_" + constant->getResultType()->getName(), table_expression->getAlias() + "." + name }); + LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "{} -> {}", missed_list.back().first, missed_list.back().second); LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Name {} Expression\n{}", name, column_name_to_node[name]->dumpTree()); } LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Table expression\n{} ", table_expression->dumpTree()); replaceColumns(query, table_expression, column_name_to_node); LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Result:\n{} ", query->dumpTree()); + + return missed_list; } Field FieldVisitorReplaceScalars::operator()(const Array & x) const diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index f4a8abe8abf..013e525832e 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -102,7 +102,9 @@ void replaceMissedSubcolumnsByConstants( const ColumnsDescription & available_columns, ASTPtr query); -void replaceMissedSubcolumnsByConstants( +using MissingObjectList = std::vector>; + +MissingObjectList replaceMissedSubcolumnsByConstants( const ColumnsDescription & expected_columns, const ColumnsDescription & available_columns, QueryTreeNodePtr & query, diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 5167ffc0e27..5bcd1ce68cb 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -151,7 +151,8 @@ void SelectStreamFactory::createForShardImpl( Shards & remote_shards, UInt32 shard_count, bool parallel_replicas_enabled, - AdditionalShardFilterGenerator shard_filter_generator) + AdditionalShardFilterGenerator shard_filter_generator, + MissingObjectList missed_list) { auto emplace_local_stream = [&]() { @@ -177,6 +178,7 @@ void SelectStreamFactory::createForShardImpl( .query = query_ast, .main_table = main_table, .header = shard_header, + .missing_object_list = std::move(missed_list), .shard_info = shard_info, .lazy = lazy, .local_delay = local_delay, @@ -299,8 +301,9 @@ void SelectStreamFactory::createForShard( auto it = objects_by_shard.find(shard_info.shard_num); QueryTreeNodePtr modified_query = query_tree; + MissingObjectList missed_list; if (it != objects_by_shard.end()) - replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); + missed_list = replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); auto query_ast = queryNodeToDistributedSelectQuery(modified_query); @@ -314,7 +317,8 @@ void SelectStreamFactory::createForShard( remote_shards, shard_count, parallel_replicas_enabled, - std::move(shard_filter_generator)); + std::move(shard_filter_generator), + std::move(missed_list)); } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 45d6ea14c01..bee7edb3c19 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -43,6 +44,8 @@ ASTPtr rewriteSelectQuery( using ColumnsDescriptionByShardNum = std::unordered_map; using AdditionalShardFilterGenerator = std::function; +using MissingObjectList = std::vector>; + class SelectStreamFactory { public: @@ -55,6 +58,8 @@ public: StorageID main_table; Block header; + MissingObjectList missing_object_list; + Cluster::ShardInfo shard_info; /// If we connect to replicas lazily. @@ -112,7 +117,8 @@ private: Shards & remote_shards, UInt32 shard_count, bool parallel_replicas_enabled, - AdditionalShardFilterGenerator shard_filter_generator); + AdditionalShardFilterGenerator shard_filter_generator, + MissingObjectList missed_list = {}); }; } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index fde2313bc15..ac507c6d555 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include "DataTypes/ObjectUtils.h" #include #include #include @@ -31,6 +33,48 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +static void addRenamingActions(Pipe & pipe, const MissingObjectList & missed_list, const Block & output_header) +{ + if (missed_list.empty()) + return; + + const auto & output_columns = output_header.getColumnsWithTypeAndName(); + std::vector indexes; + for (size_t i = 0; i < output_columns.size(); ++i) + { + bool found = false; + for (auto const & elem : missed_list) + { + if (output_columns[i].name.contains(elem.second)) + { + found = true; + break; + } + } + if (found) + indexes.push_back(i); + } + + auto dag = std::make_shared(pipe.getHeader().getColumnsWithTypeAndName()); + + for (size_t index : indexes) + { + dag->addOrReplaceInOutputs(dag->addAlias(*dag->getOutputs()[index], output_header.getByPosition(index).name)); + } + + // dag->addAliases(rename_to_apply); + + auto convert_actions = std::make_shared(dag); + pipe.addSimpleTransform([&](const Block & cur_header, Pipe::StreamType) -> ProcessorPtr + { + return std::make_shared(cur_header, convert_actions); + }); + + LOG_DEBUG(&Poco::Logger::get("addRenamingActions"), "EXPECTED:\n{}", output_header.dumpStructure()); + + LOG_DEBUG(&Poco::Logger::get("addRenamingActions"), "{}", pipe.getHeader().dumpStructure()); +} + static void addConvertingActions(Pipe & pipe, const Block & header) { if (blocksHaveEqualStructure(pipe.getHeader(), header)) @@ -216,6 +260,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); + addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); addConvertingActions(pipes.back(), output_stream->header); } @@ -297,6 +342,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); + addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); addConvertingActions(pipes.back(), output_stream->header); } } @@ -326,6 +372,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); + addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); addConvertingActions(pipes.back(), output_stream->header); } } diff --git a/tests/integration/test_distributed_type_object/test.py b/tests/integration/test_distributed_type_object/test.py index b2179af8a3f..f77e0248f02 100644 --- a/tests/integration/test_distributed_type_object/test.py +++ b/tests/integration/test_distributed_type_object/test.py @@ -59,7 +59,7 @@ def test_distributed_type_object(started_cluster): ) expected = TSV("120\n") - assert TSV(node1.query("SELECT sum(data.k2 * id) FROM dist_table")) == expected + assert TSV(node1.query("SELECT sum(data.k2 * id) FROM dist_table SETTINGS optimize_arithmetic_operations_in_aggregate_functions = 0")) == expected node1.query("TRUNCATE TABLE local_table") node2.query("TRUNCATE TABLE local_table") From 41deadda359ca02528fa6ffe9ecfed09c36b364a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 29 Feb 2024 18:37:00 +0000 Subject: [PATCH 0050/1165] Automatic style fix --- tests/integration/test_distributed_type_object/test.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_distributed_type_object/test.py b/tests/integration/test_distributed_type_object/test.py index f77e0248f02..7e6c000cb8e 100644 --- a/tests/integration/test_distributed_type_object/test.py +++ b/tests/integration/test_distributed_type_object/test.py @@ -59,7 +59,14 @@ def test_distributed_type_object(started_cluster): ) expected = TSV("120\n") - assert TSV(node1.query("SELECT sum(data.k2 * id) FROM dist_table SETTINGS optimize_arithmetic_operations_in_aggregate_functions = 0")) == expected + assert ( + TSV( + node1.query( + "SELECT sum(data.k2 * id) FROM dist_table SETTINGS optimize_arithmetic_operations_in_aggregate_functions = 0" + ) + ) + == expected + ) node1.query("TRUNCATE TABLE local_table") node2.query("TRUNCATE TABLE local_table") From a6cebad52bf4f29984db99cd4d4aa1eb41c50895 Mon Sep 17 00:00:00 2001 From: Nataly Merezhuk Date: Thu, 29 Feb 2024 16:32:29 -0500 Subject: [PATCH 0051/1165] Adds note on supported PostgreSQL versions. --- docs/en/engines/table-engines/integrations/postgresql.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index 131df1a435b..9cc4b11243e 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -8,6 +8,10 @@ sidebar_label: PostgreSQL The PostgreSQL engine allows to perform `SELECT` and `INSERT` queries on data that is stored on a remote PostgreSQL server. +:::note +Currently, only PostgreSQL versions 12 and up are supported. +::: + ## Creating a Table {#creating-a-table} ``` sql From 0f2d47e5a444bf78ffef6b2506e50079e6bb55c9 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 1 Mar 2024 10:52:44 +0100 Subject: [PATCH 0052/1165] Renamed WriteBufferFromS3TaskTracker to ThreadPoolTaskTracker --- .../ThreadPoolTaskTracker.cpp} | 2 +- .../ThreadPoolTaskTracker.h} | 6 +++--- src/Disks/IO/WriteBufferFromAzureBlobStorage.h | 2 +- src/IO/WriteBufferFromS3.cpp | 2 +- src/IO/WriteBufferFromS3.h | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) rename src/{IO/WriteBufferFromS3TaskTracker.cpp => Common/ThreadPoolTaskTracker.cpp} (99%) rename src/{IO/WriteBufferFromS3TaskTracker.h => Common/ThreadPoolTaskTracker.h} (94%) diff --git a/src/IO/WriteBufferFromS3TaskTracker.cpp b/src/Common/ThreadPoolTaskTracker.cpp similarity index 99% rename from src/IO/WriteBufferFromS3TaskTracker.cpp rename to src/Common/ThreadPoolTaskTracker.cpp index e62de261fc2..10207eb6296 100644 --- a/src/IO/WriteBufferFromS3TaskTracker.cpp +++ b/src/Common/ThreadPoolTaskTracker.cpp @@ -1,6 +1,6 @@ #include "config.h" -#include +#include "ThreadPoolTaskTracker.h" namespace ProfileEvents { diff --git a/src/IO/WriteBufferFromS3TaskTracker.h b/src/Common/ThreadPoolTaskTracker.h similarity index 94% rename from src/IO/WriteBufferFromS3TaskTracker.h rename to src/Common/ThreadPoolTaskTracker.h index 4061f084a76..d37b759a913 100644 --- a/src/IO/WriteBufferFromS3TaskTracker.h +++ b/src/Common/ThreadPoolTaskTracker.h @@ -1,10 +1,10 @@ #pragma once #include "config.h" -#include -#include "WriteBufferFromS3.h" +#include "threadPoolCallbackRunner.h" +#include "IO/WriteBufferFromS3.h" -#include +#include "logger_useful.h" #include diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index 2d11014fa2a..4897ca9a846 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include namespace Poco diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 6fc0a35672f..510d9bef4d3 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -4,8 +4,8 @@ #include "StdIStreamFromMemory.h" #include "WriteBufferFromS3.h" -#include "WriteBufferFromS3TaskTracker.h" +#include #include #include #include diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 28754d180bf..afd8b9909c1 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include From 6143986b6d79c0262f5f7dc3052ec2a3f4cfc490 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Fri, 1 Mar 2024 14:55:02 +0300 Subject: [PATCH 0053/1165] Add query test --- ...raverse_shadow_system_data_paths.reference | 3 ++ ...03000_traverse_shadow_system_data_paths.sh | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference create mode 100755 tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference new file mode 100644 index 00000000000..e8183f05f5d --- /dev/null +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference @@ -0,0 +1,3 @@ +1 +1 +1 diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh new file mode 100755 index 00000000000..a22cb200f9a --- /dev/null +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE="03000_traverse_shadow_system_data_path_table" +BACKUP="03000_traverse_shadow_system_data_path_backup" + +${CLICKHOUSE_CLIENT} --query="CREATE TABLE ${TABLE} ( + id Int64, + data String +) ENGINE=MergeTree() +ORDER BY id +SETTINGS storage_policy='s3_cache';" + +${CLICKHOUSE_CLIENT} --query="INSERT INTO ${TABLE} VALUES (0, 'data');" +${CLICKHOUSE_CLIENT} --query "SELECT count() > 0 FROM system.remote_data_paths WHERE disk_name = 's3_cache'" + +${CLICKHOUSE_CLIENT} --query="ALTER TABLE ${TABLE} FREEZE WITH NAME '${BACKUP}';" +${CLICKHOUSE_CLIENT} --query="DROP TABLE ${TABLE} SYNC;" + +${CLICKHOUSE_CLIENT} --query " + SELECT count() > 0 + FROM system.remote_data_paths + WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' + SETTINGS traverse_shadow_remote_data_paths=1;" +${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" >/dev/null +${CLICKHOUSE_CLIENT} --query " + SELECT count() == 0 + FROM system.remote_data_paths + WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' + SETTINGS traverse_shadow_remote_data_paths=1;" From a7aeb4c00f106d396364bf2a21697e329d3d284d Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 1 Mar 2024 23:44:58 +0800 Subject: [PATCH 0054/1165] Add --now option to enable and start the service --- packages/clickhouse-server.postinstall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/clickhouse-server.postinstall b/packages/clickhouse-server.postinstall index d3b49db758f..41d4405a790 100644 --- a/packages/clickhouse-server.postinstall +++ b/packages/clickhouse-server.postinstall @@ -36,7 +36,7 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then fi /bin/systemctl daemon-reload - /bin/systemctl enable clickhouse-server + /bin/systemctl enable --now clickhouse-server else # If you downgrading to version older than 1.1.54336 run: systemctl disable clickhouse-server if [ -x "/etc/init.d/clickhouse-server" ]; then From dcff866c9fa83c1d0fb755158e57096ec1a89155 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 1 Mar 2024 19:36:49 +0000 Subject: [PATCH 0055/1165] add virtual column _block_offset --- .../Algorithms/SummingSortedAlgorithm.cpp | 2 +- src/Storages/MergeTree/IMergeTreeReader.cpp | 2 +- src/Storages/MergeTree/MergeTask.cpp | 28 +++++++++---- src/Storages/MergeTree/MergeTask.h | 11 +++-- .../MergeTree/MergeTreeBlockReadUtils.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 1 + .../MergeTree/MergeTreeRangeReader.cpp | 27 +++++++++--- .../MergeTree/MergeTreeSequentialSource.cpp | 10 +++++ src/Storages/MergeTree/MergeTreeSettings.h | 3 +- src/Storages/MergeTree/MutateTask.cpp | 25 +++++++---- src/Storages/MergeTreeVirtualColumns.cpp | 4 ++ src/Storages/MergeTreeVirtualColumns.h | 7 ++++ .../03001_block_offset_column.reference | 41 +++++++++++++++++++ .../0_stateless/03001_block_offset_column.sql | 34 +++++++++++++++ 14 files changed, 167 insertions(+), 30 deletions(-) create mode 100644 tests/queries/0_stateless/03001_block_offset_column.reference create mode 100644 tests/queries/0_stateless/03001_block_offset_column.sql diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 6253d3058aa..275179e5a50 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -224,7 +224,7 @@ static SummingSortedAlgorithm::ColumnsDefinition defineColumns( const ColumnWithTypeAndName & column = header.safeGetByPosition(i); const auto * simple = dynamic_cast(column.type->getCustomName()); - if (column.name == BlockNumberColumn::name) + if (column.name == BlockNumberColumn::name || column.name == BlockOffsetColumn::name) { def.column_numbers_not_to_aggregate.push_back(i); continue; diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 30be1aa1c56..8bef26ca240 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -93,7 +93,7 @@ void IMergeTreeReader::fillVirtualColumns(Columns & columns, size_t rows) const it->name, it->type->getName(), virtual_column->type->getName()); } - if (it->name == "_part_offset") + if (it->name == "_part_offset" || it->name == BlockOffsetColumn::name) throw Exception(ErrorCodes::LOGICAL_ERROR, "Virtual column {} must be filled by range reader", it->name); Field field; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 4621314cb98..03b10034547 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -7,6 +7,7 @@ #include #include +#include "Storages/MergeTreeVirtualColumns.h" #include #include @@ -224,13 +225,11 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() ctx->need_remove_expired_values = false; ctx->force_ttl = false; - if (supportsBlockNumberColumn(global_ctx) && !global_ctx->storage_columns.contains(BlockNumberColumn::name)) - { - global_ctx->storage_columns.emplace_back(NameAndTypePair{BlockNumberColumn::name,BlockNumberColumn::type}); - global_ctx->all_column_names.emplace_back(BlockNumberColumn::name); - global_ctx->gathering_columns.emplace_back(NameAndTypePair{BlockNumberColumn::name,BlockNumberColumn::type}); - global_ctx->gathering_column_names.emplace_back(BlockNumberColumn::name); - } + if (enableBlockNumberColumn(global_ctx)) + addGatheringColumn(global_ctx, BlockNumberColumn::name, BlockNumberColumn::type); + + if (enableBlockOffsetColumn(global_ctx)) + addGatheringColumn(global_ctx, BlockOffsetColumn::name, BlockOffsetColumn::type); SerializationInfo::Settings info_settings = { @@ -296,7 +295,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() switch (global_ctx->chosen_merge_algorithm) { - case MergeAlgorithm::Horizontal : + case MergeAlgorithm::Horizontal: { global_ctx->merging_columns = global_ctx->storage_columns; global_ctx->merging_column_names = global_ctx->all_column_names; @@ -304,7 +303,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() global_ctx->gathering_column_names.clear(); break; } - case MergeAlgorithm::Vertical : + case MergeAlgorithm::Vertical: { ctx->rows_sources_uncompressed_write_buf = ctx->tmp_disk->createRawStream(); ctx->rows_sources_write_buf = std::make_unique(*ctx->rows_sources_uncompressed_write_buf); @@ -402,6 +401,17 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() return false; } +void MergeTask::addGatheringColumn(GlobalRuntimeContextPtr global_ctx, const String & name, const DataTypePtr & type) +{ + if (global_ctx->storage_columns.contains(name)) + return; + + global_ctx->storage_columns.emplace_back(name, type); + global_ctx->all_column_names.emplace_back(name); + global_ctx->gathering_columns.emplace_back(name, type); + global_ctx->gathering_column_names.emplace_back(name); +} + MergeTask::StageRuntimeContextPtr MergeTask::ExecuteAndFinalizeHorizontalPart::getContextForNextStage() { diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 28a3c671914..f7b58f818a6 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -404,12 +404,17 @@ private: Stages::const_iterator stages_iterator = stages.begin(); - /// Check for persisting block number column - static bool supportsBlockNumberColumn(GlobalRuntimeContextPtr global_ctx) + static bool enableBlockNumberColumn(GlobalRuntimeContextPtr global_ctx) { - return global_ctx->data->getSettings()->allow_experimental_block_number_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); + return global_ctx->data->getSettings()->enable_block_number_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); } + static bool enableBlockOffsetColumn(GlobalRuntimeContextPtr global_ctx) + { + return global_ctx->data->getSettings()->enable_block_offset_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); + } + + static void addGatheringColumn(GlobalRuntimeContextPtr global_ctx, const String & name, const DataTypePtr & type); }; /// FIXME diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index e84ed0a8068..7d54d3867ac 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -278,7 +278,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( .withVirtuals() .withSubcolumns(with_subcolumns); - static const NameSet columns_to_read_at_first_step = {"_part_offset"}; + static const NameSet columns_to_read_at_first_step = {"_part_offset", BlockOffsetColumn::name}; NameSet columns_from_previous_steps; auto add_step = [&](const PrewhereExprStep & step) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0b7ac39aa1b..6d23969b606 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -449,6 +449,7 @@ VirtualColumnsDescription MergeTreeData::createVirtuals(const StorageInMemoryMet desc.addPersistent(RowExistsColumn::name, RowExistsColumn::type, nullptr, "Persisted mask created by lightweight delete that show whether row exists or is deleted"); desc.addPersistent(BlockNumberColumn::name, BlockNumberColumn::type, BlockNumberColumn::codec, "Persisted original number of block that was assigned at insert"); + desc.addPersistent(BlockOffsetColumn::name, BlockOffsetColumn::type, BlockOffsetColumn::codec, "Persisted original number of row in block that was assigned at insert"); return desc; } diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index ff86ec01efa..d23ad7bfed1 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -5,6 +5,7 @@ #include #include #include +#include "Storages/MergeTreeVirtualColumns.h" #include #include #include @@ -1148,12 +1149,28 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t if (!result.rows_per_granule.empty()) result.adjustLastGranule(); - if (read_sample_block.has("_part_offset")) + bool has_part_offset = read_sample_block.has("_part_offset"); + bool has_block_offset = read_sample_block.has(BlockOffsetColumn::name); + + if (has_part_offset || has_block_offset) { - size_t pos = read_sample_block.getPositionByName("_part_offset"); - chassert(pos < result.columns.size()); - chassert(result.columns[pos] == nullptr); - result.columns[pos] = fillPartOffsetColumn(result, leading_begin_part_offset, leading_end_part_offset); + auto part_offset_column = fillPartOffsetColumn(result, leading_begin_part_offset, leading_end_part_offset); + + auto add_offset_column = [&](const auto & column_name) + { + size_t pos = read_sample_block.getPositionByName(column_name); + chassert(pos < result.columns.size()); + + /// Column may be persisted in part. + if (!result.columns[pos]) + result.columns[pos] = part_offset_column; + }; + + if (has_part_offset) + add_offset_column("_part_offset"); + + if (has_block_offset) + add_offset_column(BlockOffsetColumn::name); } return result; diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 2eb010c54ec..fb0bc617aa4 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -210,6 +210,16 @@ static void fillBlockNumberColumns( { res_columns[i] = BlockNumberColumn::type->createColumnConst(num_rows, block_number)->convertToFullColumnIfConst(); } + else if (it->name == BlockOffsetColumn::name) + { + auto column = BlockOffsetColumn::type->createColumn(); + auto & block_offset_data = assert_cast(*column).getData(); + + block_offset_data.resize(num_rows); + std::iota(block_offset_data.begin(), block_offset_data.end(), block_offset); + + res_columns[i] = std::move(column); + } } } diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 45c56a037f9..d80567a25ec 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -184,6 +184,8 @@ struct Settings; M(Bool, disable_freeze_partition_for_zero_copy_replication, true, "Disable FREEZE PARTITION query for zero copy replication.", 0) \ M(Bool, disable_detach_partition_for_zero_copy_replication, true, "Disable DETACH PARTITION query for zero copy replication.", 0) \ M(Bool, disable_fetch_partition_for_zero_copy_replication, true, "Disable FETCH PARTITION query for zero copy replication.", 0) \ + M(Bool, enable_block_number_column, false, "Enable persisting column _block_number for each row.", 0) ALIAS(allow_experimental_block_number_column) \ + M(Bool, enable_block_offset_column, false, "Enable persisting column _block_offset for each row.", 0) \ \ /** Experimental/work in progress feature. Unsafe for production. */ \ M(UInt64, part_moves_between_shards_enable, 0, "Experimental/Incomplete feature to move parts between shards. Does not take into account sharding expressions.", 0) \ @@ -192,7 +194,6 @@ struct Settings; M(String, remote_fs_zero_copy_zookeeper_path, "/clickhouse/zero_copy", "ZooKeeper path for zero-copy table-independent info.", 0) \ M(Bool, remote_fs_zero_copy_path_compatible_mode, false, "Run zero-copy in compatible mode during conversion process.", 0) \ M(Bool, cache_populated_by_fetch, false, "Only available in ClickHouse Cloud", 0) \ - M(Bool, allow_experimental_block_number_column, false, "Enable persisting column _block_number for each row.", 0) \ M(Bool, allow_experimental_replacing_merge_with_cleanup, false, "Allow experimental CLEANUP merges for ReplacingMergeTree with is_deleted column.", 0) \ \ /** Compress marks and primary key. */ \ diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index b3c36f7180b..4ba67b64c81 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -324,19 +324,26 @@ getColumnsForNewDataPart( if (!storage_columns_set.contains(RowExistsColumn::name)) { - if (deleted_mask_updated || (part_columns.has(RowExistsColumn::name) && !has_delete_command)) - { - storage_columns.emplace_back(RowExistsColumn::name, RowExistsColumn::type); - storage_columns_set.insert(RowExistsColumn::name); - } + } - if (!storage_columns_set.contains(BlockNumberColumn::name)) + auto persistent_virtuals = source_part->storage.getVirtualsDescription()->get(VirtualsKind::Persistent); + + for (const auto & [name, type] : persistent_virtuals) { - if (source_part->tryGetSerialization(BlockNumberColumn::name) != nullptr) + if (storage_columns_set.contains(name)) + continue; + + bool need_column = false; + if (name == RowExistsColumn::name) + need_column = deleted_mask_updated || (part_columns.has(name) && !has_delete_command); + else + need_column = part_columns.has(name); + + if (need_column) { - storage_columns.push_back({BlockNumberColumn::name, BlockNumberColumn::type}); - storage_columns_set.insert(BlockNumberColumn::name); + storage_columns.emplace_back(name, type); + storage_columns_set.insert(name); } } diff --git a/src/Storages/MergeTreeVirtualColumns.cpp b/src/Storages/MergeTreeVirtualColumns.cpp index 8250ceda7fa..885e46c6828 100644 --- a/src/Storages/MergeTreeVirtualColumns.cpp +++ b/src/Storages/MergeTreeVirtualColumns.cpp @@ -26,6 +26,10 @@ const String BlockNumberColumn::name = "_block_number"; const DataTypePtr BlockNumberColumn::type = std::make_shared(); const ASTPtr BlockNumberColumn::codec = getCompressionCodecDeltaLZ4(); +const String BlockOffsetColumn::name = "_block_offset"; +const DataTypePtr BlockOffsetColumn::type = std::make_shared(); +const ASTPtr BlockOffsetColumn::codec = getCompressionCodecDeltaLZ4(); + Field getFieldForConstVirtualColumn(const String & column_name, const IMergeTreeDataPart & part) { if (column_name == RowExistsColumn::name) diff --git a/src/Storages/MergeTreeVirtualColumns.h b/src/Storages/MergeTreeVirtualColumns.h index 24721bf1ad1..cd9fe544ed8 100644 --- a/src/Storages/MergeTreeVirtualColumns.h +++ b/src/Storages/MergeTreeVirtualColumns.h @@ -21,6 +21,13 @@ struct BlockNumberColumn static const ASTPtr codec; }; +struct BlockOffsetColumn +{ + static const String name; + static const DataTypePtr type; + static const ASTPtr codec; +}; + Field getFieldForConstVirtualColumn(const String & column_name, const IMergeTreeDataPart & part); } diff --git a/tests/queries/0_stateless/03001_block_offset_column.reference b/tests/queries/0_stateless/03001_block_offset_column.reference new file mode 100644 index 00000000000..fba88974380 --- /dev/null +++ b/tests/queries/0_stateless/03001_block_offset_column.reference @@ -0,0 +1,41 @@ +*** BEFORE MUTATION BEFORE MERGE *** +1 1 1 0 all_1_1_0 +2 2 2 0 all_2_2_0 +3 3 1 1 all_1_1_0 +4 4 1 2 all_1_1_0 +5 5 2 1 all_2_2_0 +6 6 2 2 all_2_2_0 +*** AFTER MUTATION BEFORE MERGE *** +1 0 1 0 all_1_1_0_3 +2 0 2 0 all_2_2_0_3 +3 0 1 1 all_1_1_0_3 +4 4 1 2 all_1_1_0_3 +5 5 2 1 all_2_2_0_3 +6 6 2 2 all_2_2_0_3 +*** AFTER MUTATION AFTER MERGE *** +1 0 1 0 all_1_2_1_3 +2 0 2 0 all_1_2_1_3 +3 0 1 1 all_1_2_1_3 +4 4 1 2 all_1_2_1_3 +5 5 2 1 all_1_2_1_3 +6 6 2 2 all_1_2_1_3 +*** AFTER MUTATION AFTER MERGE , NEW BLOCK *** +1 0 1 0 all_1_2_1_3 +2 0 2 0 all_1_2_1_3 +3 0 1 1 all_1_2_1_3 +4 4 1 2 all_1_2_1_3 +5 5 2 1 all_1_2_1_3 +6 6 2 2 all_1_2_1_3 +7 7 4 0 all_4_4_0 +8 8 4 1 all_4_4_0 +9 9 4 2 all_4_4_0 +*** AFTER MUTATION AFTER MERGE , NEW BLOCK MERGED *** +1 0 1 0 all_1_4_2_3 +2 0 2 0 all_1_4_2_3 +3 0 1 1 all_1_4_2_3 +4 4 1 2 all_1_4_2_3 +5 5 2 1 all_1_4_2_3 +6 6 2 2 all_1_4_2_3 +7 7 4 0 all_1_4_2_3 +8 8 4 1 all_1_4_2_3 +9 9 4 2 all_1_4_2_3 diff --git a/tests/queries/0_stateless/03001_block_offset_column.sql b/tests/queries/0_stateless/03001_block_offset_column.sql new file mode 100644 index 00000000000..0ef2b58f77b --- /dev/null +++ b/tests/queries/0_stateless/03001_block_offset_column.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test (id UInt32, a UInt32) ENGINE = MergeTree +ORDER BY id +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; + +INSERT INTO test(id,a) VALUES (1,1),(3,3),(4,4); +INSERT INTO test(id,a) VALUES (2,2),(5,5),(6,6); + +SELECT '*** BEFORE MUTATION BEFORE MERGE ***'; +SELECT id,a,_block_number,_block_offset,_part from test ORDER BY id; + +set mutations_sync=1; +ALTER TABLE test UPDATE a=0 WHERE id<4; + +SELECT '*** AFTER MUTATION BEFORE MERGE ***'; +SELECT id,a,_block_number,_block_offset,_part from test ORDER BY id; + +OPTIMIZE TABLE test FINAL; + +SELECT '*** AFTER MUTATION AFTER MERGE ***'; +SELECT *,_block_number,_block_offset,_part from test ORDER BY id; + +INSERT INTO test(id,a) VALUES (7,7),(8,8),(9,9); + +SELECT '*** AFTER MUTATION AFTER MERGE , NEW BLOCK ***'; +SELECT *,_block_number,_block_offset,_part from test ORDER BY id; + +OPTIMIZE TABLE test FINAL; + +SELECT '*** AFTER MUTATION AFTER MERGE , NEW BLOCK MERGED ***'; +SELECT *,_block_number,_block_offset,_part from test ORDER BY id; + +DROP TABLE test; \ No newline at end of file From dd400dedf247fb6c6451d4ab3e338ed53e0a7049 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 1 Mar 2024 19:51:14 +0000 Subject: [PATCH 0056/1165] add virtual column _data_version --- src/Storages/MergeTree/MergeTask.cpp | 1 - src/Storages/MergeTree/MergeTreeData.cpp | 3 ++- .../MergeTree/MergeTreeRangeReader.cpp | 2 +- src/Storages/MergeTreeVirtualColumns.cpp | 3 +++ .../03001_data_version_column.reference | 5 +++++ .../0_stateless/03001_data_version_column.sql | 20 +++++++++++++++++++ 6 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/03001_data_version_column.reference create mode 100644 tests/queries/0_stateless/03001_data_version_column.sql diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 03b10034547..12733e19870 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -7,7 +7,6 @@ #include #include -#include "Storages/MergeTreeVirtualColumns.h" #include #include diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6d23969b606..9c25b82ede7 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -440,6 +440,7 @@ VirtualColumnsDescription MergeTreeData::createVirtuals(const StorageInMemoryMet desc.addEphemeral("_partition_id", std::make_shared(std::make_shared()), "Name of partition"); desc.addEphemeral("_sample_factor", std::make_shared(), "Sample factor (from the query)"); desc.addEphemeral("_part_offset", std::make_shared(), "Number of row in the part"); + desc.addEphemeral("_data_version", std::make_shared(), "Data version of mutation of part"); if (metadata.hasPartitionKey()) { @@ -1027,7 +1028,7 @@ void MergeTreeData::MergingParams::check(const StorageInMemoryMetadata & metadat /// TODO Checks for Graphite mode. } -const Names MergeTreeData::virtuals_useful_for_filter = {"_part", "_partition_id", "_part_uuid", "_partition_value"}; +const Names MergeTreeData::virtuals_useful_for_filter = {"_part", "_partition_id", "_part_uuid", "_partition_value", "_data_version"}; Block MergeTreeData::getHeaderWithVirtualsForFilter() const { diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index d23ad7bfed1..ae6722ebce7 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -1,11 +1,11 @@ #include #include +#include #include #include #include #include #include -#include "Storages/MergeTreeVirtualColumns.h" #include #include #include diff --git a/src/Storages/MergeTreeVirtualColumns.cpp b/src/Storages/MergeTreeVirtualColumns.cpp index 885e46c6828..e67ba811df4 100644 --- a/src/Storages/MergeTreeVirtualColumns.cpp +++ b/src/Storages/MergeTreeVirtualColumns.cpp @@ -47,6 +47,9 @@ Field getFieldForConstVirtualColumn(const String & column_name, const IMergeTree if (column_name == "_partition_id") return part.info.partition_id; + if (column_name == "_data_version") + return part.info.getDataVersion(); + if (column_name == "_partition_value") return Tuple(part.partition.value.begin(), part.partition.value.end()); diff --git a/tests/queries/0_stateless/03001_data_version_column.reference b/tests/queries/0_stateless/03001_data_version_column.reference new file mode 100644 index 00000000000..cd85ce9cc63 --- /dev/null +++ b/tests/queries/0_stateless/03001_data_version_column.reference @@ -0,0 +1,5 @@ +all_1_1_0 1 1 1 +all_2_2_0 2 2 2 +all_1_1_0_3 3 1 100 +all_2_2_0_3 3 2 200 +all_4_4_0 4 3 3 diff --git a/tests/queries/0_stateless/03001_data_version_column.sql b/tests/queries/0_stateless/03001_data_version_column.sql new file mode 100644 index 00000000000..6cb6b192326 --- /dev/null +++ b/tests/queries/0_stateless/03001_data_version_column.sql @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS t_data_version; + +CREATE TABLE t_data_version (a UInt64, b UInt64) ENGINE = MergeTree ORDER BY a; + +INSERT INTO t_data_version VALUES (1, 1); +INSERT INTO t_data_version VALUES (2, 2); + +SELECT _part, _data_version, * FROM t_data_version ORDER BY a; + +ALTER TABLE t_data_version UPDATE b = a * 100 WHERE 1 SETTINGS mutations_sync = 2; + +SELECT _part, _data_version, * FROM t_data_version ORDER BY a; + +INSERT INTO t_data_version VALUES (3, 3); + +-- Check part pruning. +SELECT _part, _data_version, * FROM t_data_version WHERE _data_version = 4 ORDER BY a SETTINGS max_rows_to_read = 1; + +DROP TABLE t_data_version; + From 81185815a48b36d344bda623dd175c30e9b87ba3 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Mon, 4 Mar 2024 14:09:31 +0300 Subject: [PATCH 0057/1165] Update settings_changes_history --- src/Core/SettingsChangesHistory.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 4805df46d9b..b8793f437d8 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -85,6 +85,9 @@ namespace SettingsChangesHistory /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) static std::map settings_changes_history = { + {"24.3", { + {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."}, + }}, {"24.2", { {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"}, {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"}, From fbdc5e305365e9d93b86ed47144ffb13c1ce70c1 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Mon, 4 Mar 2024 17:16:51 +0300 Subject: [PATCH 0058/1165] Ignore flaky fail of system unfreeze --- .../0_stateless/03000_traverse_shadow_system_data_paths.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh index a22cb200f9a..2905d7801ca 100755 --- a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh @@ -26,7 +26,7 @@ ${CLICKHOUSE_CLIENT} --query " FROM system.remote_data_paths WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' SETTINGS traverse_shadow_remote_data_paths=1;" -${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" >/dev/null +${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null || true ${CLICKHOUSE_CLIENT} --query " SELECT count() == 0 FROM system.remote_data_paths From aa43885ac81924a73e9a151a550e7c1af43d23e2 Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 5 Mar 2024 10:57:25 +0800 Subject: [PATCH 0059/1165] [improve] add check the remaining disk size before copying --- src/Storages/MergeTree/MergeTreeData.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 849ceb1b66d..d8680958c21 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7160,14 +7160,18 @@ std::pair MergeTreeData::cloneAn { try { + auto reservation_space = src_part_storage->reserve(src_part->getBytesOnDisk()); + if (!reservation_space) { + throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Not enough space on disk."); + } dst_part_storage = src_part_storage->clonePart(this->getRelativeDataPath(), tmp_dst_part_name, disk, read_settings, write_settings, {}, {}); copy_successful = true; break; } - catch (...) + catch (Exception & e) { - LOG_TRACE(&Poco::Logger::get("MergeTreeData"), "Clone part on disk {} fail", disk->getName()); + LOG_TRACE(&Poco::Logger::get("MergeTreeData"), "Clone part on disk {} fail: {}", disk->getName(), e.what()); } } if (!copy_successful) @@ -7291,6 +7295,9 @@ std::pair MergeTreeData::cloneAn { try { + auto reservation_space = src_part_storage->reserve(src_part->getBytesOnDisk()); + if (!reservation_space) + throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Not enough space on disk."); dst_part_storage = src_part_storage->clonePart(this->getRelativeDataPath(), tmp_dst_part_name, disk, read_settings, write_settings, {}, {}); copy_successful = true; From 758a75c1b46fa27a88e3dcf6e70a18dcf41d62ef Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Tue, 5 Mar 2024 09:53:30 +0300 Subject: [PATCH 0060/1165] Fix flaky test. Fix clang-tidy warning --- src/Disks/IDisk.h | 7 +++---- src/Disks/ObjectStorages/DiskObjectStorage.h | 2 +- src/Storages/System/StorageSystemRemoteDataPaths.cpp | 4 ++-- .../0_stateless/03000_traverse_shadow_system_data_paths.sh | 7 +------ 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 62b02938d1a..fcc92db7b96 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -321,11 +321,10 @@ public: }; virtual void getRemotePathsRecursive( - const String &, - std::vector &, - const std::function & /* skip_predicate */ = {}) + const String &, std::vector &, const std::function & /* skip_predicate */) { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, "Method `getRemotePathsRecursive() not implemented for disk: {}`", getDataSourceDescription().toString()); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index d7af656bea3..9f11c0ed02e 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -51,7 +51,7 @@ public: void getRemotePathsRecursive( const String & local_path, std::vector & paths_map, - const std::function & skip_predicate = {}) override; + const std::function & skip_predicate) override; const std::string & getCacheName() const override { return object_storage->getCacheName(); } diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index 708c1369965..a6263f18492 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -58,8 +58,8 @@ Pipe StorageSystemRemoteDataPaths::read( if (disk->isRemote()) { std::vector remote_paths_by_local_path; - disk->getRemotePathsRecursive("store", remote_paths_by_local_path); - disk->getRemotePathsRecursive("data", remote_paths_by_local_path); + disk->getRemotePathsRecursive("store", remote_paths_by_local_path, /* skip_predicate = */ {}); + disk->getRemotePathsRecursive("data", remote_paths_by_local_path, /* skip_predicate = */ {}); if (context->getSettingsRef().traverse_shadow_remote_data_paths) disk->getRemotePathsRecursive( "shadow", diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh index 2905d7801ca..a1d4b9bba46 100755 --- a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh @@ -26,9 +26,4 @@ ${CLICKHOUSE_CLIENT} --query " FROM system.remote_data_paths WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' SETTINGS traverse_shadow_remote_data_paths=1;" -${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null || true -${CLICKHOUSE_CLIENT} --query " - SELECT count() == 0 - FROM system.remote_data_paths - WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' - SETTINGS traverse_shadow_remote_data_paths=1;" +${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null From df80c8c9f6ee0939cc6e6e05f3e951511a20f476 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Tue, 5 Mar 2024 10:43:48 +0300 Subject: [PATCH 0061/1165] Update test reference --- .../03000_traverse_shadow_system_data_paths.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference index e8183f05f5d..6ed281c757a 100644 --- a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference @@ -1,3 +1,2 @@ 1 1 -1 From e789d15948eaec3eaa9a8604e24d2f6ed7b60db5 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 5 Mar 2024 16:06:25 +0800 Subject: [PATCH 0062/1165] optimize insertmanyfrom of nullable(number) or nullable(string) --- src/Columns/ColumnDecimal.h | 7 +++++++ src/Columns/ColumnNullable.cpp | 8 ++++++++ src/Columns/ColumnNullable.h | 1 + src/Columns/ColumnString.cpp | 21 +++++++++++++++++++++ src/Columns/ColumnString.h | 2 ++ 5 files changed, 39 insertions(+) diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 7ca01a8342c..e0ea26744dc 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -56,6 +56,13 @@ public: void shrinkToFit() override { data.shrink_to_fit(); } void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast(src).getData()[n]); } + + void insertManyFrom(const IColumn & src, size_t position, size_t length) override + { + ValueType v = assert_cast(src).getData()[position]; + data.resize_fill(data.size() + length, v); + } + void insertData(const char * src, size_t /*length*/) override; void insertDefault() override { data.push_back(T()); } void insertManyDefaults(size_t length) override { data.resize_fill(data.size() + length); } diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 1d11827ac97..fa5fdfb8c21 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -231,6 +231,14 @@ void ColumnNullable::insertFrom(const IColumn & src, size_t n) getNullMapData().push_back(src_concrete.getNullMapData()[n]); } + +void ColumnNullable::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + const ColumnNullable & src_concrete = assert_cast(src); + getNestedColumn().insertManyFrom(src_concrete.getNestedColumn(), position, length); + getNullMapColumn().insertManyFrom(src_concrete.getNullMapColumn(), position, length); +} + void ColumnNullable::insertFromNotNullable(const IColumn & src, size_t n) { getNestedColumn().insertFrom(src, n); diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index b4aef8e08fa..ef4bf4fa41b 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -69,6 +69,7 @@ public: void insert(const Field & x) override; bool tryInsert(const Field & x) override; void insertFrom(const IColumn & src, size_t n) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertFromNotNullable(const IColumn & src, size_t n); void insertRangeFromNotNullable(const IColumn & src, size_t start, size_t length); diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index b9128372cea..f3c7ac1bf0c 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -38,6 +38,27 @@ ColumnString::ColumnString(const ColumnString & src) last_offset, chars.size()); } +void ColumnString::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + const ColumnString & src_concrete = assert_cast(src); + const UInt8 * src_buf = &src_concrete.chars[src_concrete.offsets[position - 1]]; + const size_t src_buf_size + = src_concrete.offsets[position] - src_concrete.offsets[position - 1]; /// -1th index is Ok, see PaddedPODArray. + + const size_t old_size = chars.size(); + const size_t new_size = old_size + src_buf_size * length; + chars.resize(new_size); + + const size_t old_rows = offsets.size(); + offsets.resize(old_rows + length); + + for (size_t current_offset = old_size; current_offset < new_size; current_offset += src_buf_size) + memcpySmallAllowReadWriteOverflow15(&chars[current_offset], src_buf, src_buf_size); + + for (size_t i = 0, current_offset = old_size + src_buf_size; i < length; ++i, current_offset += src_buf_size) + offsets[old_rows + i] = current_offset; +} + MutableColumnPtr ColumnString::cloneResized(size_t to_size) const { diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 04aa1849187..2d1d69ced73 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -160,6 +160,8 @@ public: } } + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; + void insertData(const char * pos, size_t length) override { const size_t old_size = chars.size(); From 580fd4ba080df6e29c59b785b1fca0eea76e649c Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Tue, 5 Mar 2024 10:43:48 +0300 Subject: [PATCH 0063/1165] Update test reference --- .../0_stateless/03000_traverse_shadow_system_data_paths.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh index a1d4b9bba46..98575540923 100755 --- a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh @@ -26,4 +26,4 @@ ${CLICKHOUSE_CLIENT} --query " FROM system.remote_data_paths WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' SETTINGS traverse_shadow_remote_data_paths=1;" -${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null +${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null || true From a109952960acac12790cffde030062ec60208994 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 5 Mar 2024 22:08:36 +0800 Subject: [PATCH 0064/1165] dev columnstring --- src/Columns/ColumnArray.cpp | 83 +++++++++++++++++++++++++++++++ src/Columns/ColumnArray.h | 9 ++++ src/Columns/ColumnConst.h | 2 + src/Columns/ColumnFixedString.cpp | 14 ++++++ src/Columns/ColumnFixedString.h | 2 + 5 files changed, 110 insertions(+) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 7b268b80116..b620da81ae8 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -347,6 +347,89 @@ void ColumnArray::insertFrom(const IColumn & src_, size_t n) getOffsets().push_back(getOffsets().back() + size); } +template +void ColumnArray::insertManyFromNumber(const ColumnArray & src, size_t position, size_t length) +{ + using ColVecType = ColumnVectorOrDecimal; + size_t src_size = src.sizeAt(position); + size_t src_offset = src.offsetAt(position); + + const typename ColVecType::Container & src_data = typeid_cast(src.getData()).getData(); + typename ColVecType::Container & data_ref = typeid_cast(getData()).getData(); + size_t old_size = data_ref.size(); + size_t new_size = old_size + src_size * length; + data_ref.resize(new_size); + for (size_t i = 0, offset = old_size; i < length; ++i, offset += src_size) + memcpy(&data_ref[offset], &src_data[src_offset], src_size * sizeof(T)); +} + +void ColumnArray::insertManyFromString(const ColumnArray & src, size_t position, size_t length) +{ + size_t src_size = src.sizeAt(position); + size_t src_offset = src.offsetAt(position); + + const auto & src_string = typeid_cast(src.getData()); + const auto & src_chars = src_string.getChars(); + const auto & src_string_offsets = src_string.getOffsets(); + auto & dst_string = typeid_cast(getData()); + auto & dst_chars = dst_string.getChars(); + auto & dst_string_offsets = dst_string.getOffsets(); + + /// Each row may have multiple strings, copy them to dst_chars and update dst_offsets + size_t old_size = dst_string_offsets.size(); + size_t new_size = old_size + src_size * length; + dst_string_offsets.resize(new_size); + size_t dst_string_offset = dst_chars.size(); + for (size_t i = 0; i < length; ++i) + { + for (size_t j = 0; j < src_size; ++j) + { + size_t nested_offset = src_string_offsets[src_offset + j - 1]; + size_t nested_length = src_string_offsets[src_offset + j] - nested_offset; + + dst_string_offset += nested_length; + dst_string_offsets[old_size + i * src_size + j] = dst_string_offset; + } + } + + size_t chars_to_copy = src_string_offsets[src_offset + src_size - 1] - src_string_offsets[src_offset - 1]; + dst_chars.resize(dst_chars.size() + chars_to_copy * length); + for (size_t dst_offset = old_size; dst_offset < new_size; dst_offset += src_size) + memcpy(&dst_chars[dst_string_offsets[dst_offset - 1]], &src_chars[src_string_offsets[src_offset - 1]], chars_to_copy); +} + +void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, size_t length) +{ + +} +void ColumnArray::insertManyFromNullable(const ColumnArray & src, size_t position, size_t length) +{ + +} +void ColumnArray::insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length) +{ + size_t src_size = src.sizeAt(position); + size_t src_offset = src.offsetAt(position); + const auto & src_data = src.getData(); + size_t new_size = data->size() + src_size * length; + data->reserve(new_size); + for (size_t i = 0; i < length; ++i) + data->insertRangeFrom(src_data, src_offset, src_size); +} + +void ColumnArray::insertManyFrom(const IColumn & src_, size_t position, size_t length) +{ + /// First fill offsets + const ColumnArray & src = assert_cast(src_); + size_t src_size = src.sizeAt(position); + auto & offsets_ref = getOffsets(); + size_t old_rows = offsets_ref.size(); + size_t new_rows = old_rows + length; + size_t old_size = offsets_ref.back(); + offsets_ref.resize(new_rows); + for (size_t i = 0, offset = old_size + src_size; i < length; ++i, offset += src_size) + offsets_ref[old_rows + i] = offset; +} void ColumnArray::insertDefault() { diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 230d8830265..73d632a38b9 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -88,6 +88,7 @@ public: void insert(const Field & x) override; bool tryInsert(const Field & x) override; void insertFrom(const IColumn & src_, size_t n) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertDefault() override; void popBack(size_t n) override; ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; @@ -213,6 +214,14 @@ private: ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const; ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const; + /// Specializations for insertManyFrom + template + void insertManyFromNumber(const ColumnArray & src, size_t position, size_t length); + void insertManyFromString(const ColumnArray & src, size_t position, size_t length); + void insertManyFromTuple(const ColumnArray & src, size_t position, size_t length); + void insertManyFromNullable(const ColumnArray & src, size_t position, size_t length); + void insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length); + int compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint, const Collator * collator=nullptr) const; }; diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index 990b7189fa3..4a3d40ca0d2 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -150,6 +150,8 @@ public: ++s; } + void insertManyFrom(const IColumn & /*src*/, size_t /* position */, size_t length) override { s += length; } + void insertDefault() override { ++s; diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index e460c84d696..b55f68d4687 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -85,6 +85,20 @@ void ColumnFixedString::insertFrom(const IColumn & src_, size_t index) memcpySmallAllowReadWriteOverflow15(chars.data() + old_size, &src.chars[n * index], n); } +void ColumnFixedString::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + const ColumnFixedString & src_concrete = assert_cast(src); + if (n != src_concrete.getN()) + throw Exception(ErrorCodes::SIZE_OF_FIXED_STRING_DOESNT_MATCH, "Size of FixedString doesn't match"); + + const size_t old_size = chars.size(); + const size_t new_size = old_size + n * length; + chars.resize(new_size); + + for (size_t offset = old_size; offset < new_size; offset += n) + memcpySmallAllowReadWriteOverflow15(&chars[offset], &src_concrete.chars[n * position], n); +} + void ColumnFixedString::insertData(const char * pos, size_t length) { if (length > n) diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index f40e1356b27..56d42e8b34e 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -100,6 +100,8 @@ public: void insertFrom(const IColumn & src_, size_t index) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; + void insertData(const char * pos, size_t length) override; void insertDefault() override From d529389522311e7bca11a3beebc07e0439efcfb4 Mon Sep 17 00:00:00 2001 From: Zach Naimon Date: Tue, 5 Mar 2024 14:20:20 -0500 Subject: [PATCH 0065/1165] Add support for 'START TRANSACTION' syntax --- src/Parsers/ParserTransactionControl.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Parsers/ParserTransactionControl.cpp b/src/Parsers/ParserTransactionControl.cpp index da593170002..fc3077bb0b6 100644 --- a/src/Parsers/ParserTransactionControl.cpp +++ b/src/Parsers/ParserTransactionControl.cpp @@ -14,6 +14,8 @@ bool ParserTransactionControl::parseImpl(Pos & pos, ASTPtr & node, Expected & ex if (ParserKeyword("BEGIN TRANSACTION").ignore(pos, expected)) action = ASTTransactionControl::BEGIN; + else if (ParserKeyword("START TRANSACTION").ignore(pos, expected)) + action = ASTTransactionControl::BEGIN; else if (ParserKeyword("COMMIT").ignore(pos, expected)) action = ASTTransactionControl::COMMIT; else if (ParserKeyword("ROLLBACK").ignore(pos, expected)) From d596de73847c712d766e0f8598ce43ed078ec968 Mon Sep 17 00:00:00 2001 From: Zach Naimon Date: Tue, 5 Mar 2024 14:25:48 -0500 Subject: [PATCH 0066/1165] update documentation --- docs/en/sql-reference/transactions.md | 42 +++++++++++++++++++++------ 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/transactions.md b/docs/en/sql-reference/transactions.md index cb89a091d68..b9c9afc20f9 100644 --- a/docs/en/sql-reference/transactions.md +++ b/docs/en/sql-reference/transactions.md @@ -1,26 +1,29 @@ --- slug: /en/guides/developer/transactional --- + # Transactional (ACID) support -## Case 1: INSERT into one partition, of one table, of the MergeTree* family +## Case 1: INSERT into one partition, of one table, of the MergeTree\* family This is transactional (ACID) if the inserted rows are packed and inserted as a single block (see Notes): + - Atomic: an INSERT succeeds or is rejected as a whole: if a confirmation is sent to the client, then all rows were inserted; if an error is sent to the client, then no rows were inserted. - Consistent: if there are no table constraints violated, then all rows in an INSERT are inserted and the INSERT succeeds; if constraints are violated, then no rows are inserted. - Isolated: concurrent clients observe a consistent snapshot of the table–the state of the table either as it was before the INSERT attempt, or after the successful INSERT; no partial state is seen - Durable: a successful INSERT is written to the filesystem before answering to the client, on a single replica or multiple replicas (controlled by the `insert_quorum` setting), and ClickHouse can ask the OS to sync the filesystem data on the storage media (controlled by the `fsync_after_insert` setting). - INSERT into multiple tables with one statement is possible if materialized views are involved (the INSERT from the client is to a table which has associate materialized views). -## Case 2: INSERT into multiple partitions, of one table, of the MergeTree* family +## Case 2: INSERT into multiple partitions, of one table, of the MergeTree\* family Same as Case 1 above, with this detail: + - If table has many partitions and INSERT covers many partitions–then insertion into every partition is transactional on its own - -## Case 3: INSERT into one distributed table of the MergeTree* family +## Case 3: INSERT into one distributed table of the MergeTree\* family Same as Case 1 above, with this detail: + - INSERT into Distributed table is not transactional as a whole, while insertion into every shard is transactional ## Case 4: Using a Buffer table @@ -30,9 +33,11 @@ Same as Case 1 above, with this detail: ## Case 5: Using async_insert Same as Case 1 above, with this detail: + - atomicity is ensured even if `async_insert` is enabled and `wait_for_async_insert` is set to 1 (the default), but if `wait_for_async_insert` is set to 0, then atomicity is not ensured. ## Notes + - rows inserted from the client in some data format are packed into a single block when: - the insert format is row-based (like CSV, TSV, Values, JSONEachRow, etc) and the data contains less then `max_insert_block_size` rows (~1 000 000 by default) or less then `min_chunk_bytes_for_parallel_parsing` bytes (10 MB by default) in case of parallel parsing is used (enabled by default) - the insert format is column-based (like Native, Parquet, ORC, etc) and the data contains only one block of data @@ -61,8 +66,9 @@ In addition to the functionality described at the top of this document, ClickHou ``` ### Notes + - This is an experimental feature, and changes should be expected. -- If an exception occurs during a transaction, you cannot commit the transaction. This includes all exceptions, including `UNKNOWN_FUNCTION` exceptions caused by typos. +- If an exception occurs during a transaction, you cannot commit the transaction. This includes all exceptions, including `UNKNOWN_FUNCTION` exceptions caused by typos. - Nested transactions are not supported; finish the current transaction and start a new one instead ### Configuration @@ -80,7 +86,7 @@ These examples are with a single node ClickHouse server with ClickHouse Keeper e #### Basic configuration for a single ClickHouse server node with ClickHouse Keeper enabled :::note -See the [deployment](docs/en/deployment-guides/terminology.md) documentation for details on deploying ClickHouse server and a proper quorum of ClickHouse Keeper nodes. The configuration shown here is for experimental purposes. +See the [deployment](docs/en/deployment-guides/terminology.md) documentation for details on deploying ClickHouse server and a proper quorum of ClickHouse Keeper nodes. The configuration shown here is for experimental purposes. ::: ```xml title=/etc/clickhouse-server/config.d/config.xml @@ -127,17 +133,19 @@ See the [deployment](docs/en/deployment-guides/terminology.md) documentation for #### Verify that experimental transactions are enabled -Issue a `BEGIN TRANSACTION` followed by a `ROLLBACK` to verify that experimental transactions are enabled, and that ClickHouse Keeper is enabled as it is used to track transactions. +Issue a `BEGIN TRANSACTION` or `START TRANSACTION` followed by a `ROLLBACK` to verify that experimental transactions are enabled, and that ClickHouse Keeper is enabled as it is used to track transactions. ```sql BEGIN TRANSACTION ``` + ```response Ok. ``` :::tip If you see the following error, then check your configuration file to make sure that `allow_experimental_transactions` is set to `1` (or any value other than `0` or `false`). + ``` Code: 48. DB::Exception: Received from localhost:9000. DB::Exception: Transactions are not supported. @@ -145,15 +153,18 @@ DB::Exception: Transactions are not supported. ``` You can also check ClickHouse Keeper by issuing + ``` echo ruok | nc localhost 9181 ``` + ClickHouse Keeper should respond with `imok`. ::: ```sql ROLLBACK ``` + ```response Ok. ``` @@ -161,7 +172,7 @@ Ok. #### Create a table for testing :::tip -Creation of tables is not transactional. Run this DDL query outside of a transaction. +Creation of tables is not transactional. Run this DDL query outside of a transaction. ::: ```sql @@ -172,6 +183,7 @@ CREATE TABLE mergetree_table ENGINE = MergeTree ORDER BY n ``` + ```response Ok. ``` @@ -181,6 +193,7 @@ Ok. ```sql BEGIN TRANSACTION ``` + ```response Ok. ``` @@ -188,6 +201,7 @@ Ok. ```sql INSERT INTO mergetree_table FORMAT Values (10) ``` + ```response Ok. ``` @@ -196,11 +210,13 @@ Ok. SELECT * FROM mergetree_table ``` + ```response ┌──n─┐ │ 10 │ └────┘ ``` + :::note You can query the table from within a transaction and see that the row was inserted even though it has not yet been committed. ::: @@ -208,16 +224,20 @@ You can query the table from within a transaction and see that the row was inser #### Rollback the transaction, and query the table again Verify that the transaction is rolled back: + ```sql ROLLBACK ``` + ```response Ok. ``` + ```sql SELECT * FROM mergetree_table ``` + ```response Ok. @@ -229,6 +249,7 @@ Ok. ```sql BEGIN TRANSACTION ``` + ```response Ok. ``` @@ -236,6 +257,7 @@ Ok. ```sql INSERT INTO mergetree_table FORMAT Values (42) ``` + ```response Ok. ``` @@ -243,6 +265,7 @@ Ok. ```sql COMMIT ``` + ```response Ok. Elapsed: 0.002 sec. ``` @@ -251,6 +274,7 @@ Ok. Elapsed: 0.002 sec. SELECT * FROM mergetree_table ``` + ```response ┌──n─┐ │ 42 │ @@ -267,6 +291,7 @@ SELECT * FROM system.transactions FORMAT Vertical ``` + ```response Row 1: ────── @@ -280,4 +305,3 @@ state: RUNNING ## More Details See this [meta issue](https://github.com/ClickHouse/ClickHouse/issues/48794) to find much more extensive tests and to keep up to date with the progress. - From 34d327a08cc2de20510eb429c2da30b4d135a3f0 Mon Sep 17 00:00:00 2001 From: Zach Naimon Date: Tue, 5 Mar 2024 14:26:57 -0500 Subject: [PATCH 0067/1165] Revert "update documentation" This reverts commit d596de73847c712d766e0f8598ce43ed078ec968. --- docs/en/sql-reference/transactions.md | 42 ++++++--------------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/docs/en/sql-reference/transactions.md b/docs/en/sql-reference/transactions.md index b9c9afc20f9..cb89a091d68 100644 --- a/docs/en/sql-reference/transactions.md +++ b/docs/en/sql-reference/transactions.md @@ -1,29 +1,26 @@ --- slug: /en/guides/developer/transactional --- - # Transactional (ACID) support -## Case 1: INSERT into one partition, of one table, of the MergeTree\* family +## Case 1: INSERT into one partition, of one table, of the MergeTree* family This is transactional (ACID) if the inserted rows are packed and inserted as a single block (see Notes): - - Atomic: an INSERT succeeds or is rejected as a whole: if a confirmation is sent to the client, then all rows were inserted; if an error is sent to the client, then no rows were inserted. - Consistent: if there are no table constraints violated, then all rows in an INSERT are inserted and the INSERT succeeds; if constraints are violated, then no rows are inserted. - Isolated: concurrent clients observe a consistent snapshot of the table–the state of the table either as it was before the INSERT attempt, or after the successful INSERT; no partial state is seen - Durable: a successful INSERT is written to the filesystem before answering to the client, on a single replica or multiple replicas (controlled by the `insert_quorum` setting), and ClickHouse can ask the OS to sync the filesystem data on the storage media (controlled by the `fsync_after_insert` setting). - INSERT into multiple tables with one statement is possible if materialized views are involved (the INSERT from the client is to a table which has associate materialized views). -## Case 2: INSERT into multiple partitions, of one table, of the MergeTree\* family +## Case 2: INSERT into multiple partitions, of one table, of the MergeTree* family Same as Case 1 above, with this detail: - - If table has many partitions and INSERT covers many partitions–then insertion into every partition is transactional on its own -## Case 3: INSERT into one distributed table of the MergeTree\* family + +## Case 3: INSERT into one distributed table of the MergeTree* family Same as Case 1 above, with this detail: - - INSERT into Distributed table is not transactional as a whole, while insertion into every shard is transactional ## Case 4: Using a Buffer table @@ -33,11 +30,9 @@ Same as Case 1 above, with this detail: ## Case 5: Using async_insert Same as Case 1 above, with this detail: - - atomicity is ensured even if `async_insert` is enabled and `wait_for_async_insert` is set to 1 (the default), but if `wait_for_async_insert` is set to 0, then atomicity is not ensured. ## Notes - - rows inserted from the client in some data format are packed into a single block when: - the insert format is row-based (like CSV, TSV, Values, JSONEachRow, etc) and the data contains less then `max_insert_block_size` rows (~1 000 000 by default) or less then `min_chunk_bytes_for_parallel_parsing` bytes (10 MB by default) in case of parallel parsing is used (enabled by default) - the insert format is column-based (like Native, Parquet, ORC, etc) and the data contains only one block of data @@ -66,9 +61,8 @@ In addition to the functionality described at the top of this document, ClickHou ``` ### Notes - - This is an experimental feature, and changes should be expected. -- If an exception occurs during a transaction, you cannot commit the transaction. This includes all exceptions, including `UNKNOWN_FUNCTION` exceptions caused by typos. +- If an exception occurs during a transaction, you cannot commit the transaction. This includes all exceptions, including `UNKNOWN_FUNCTION` exceptions caused by typos. - Nested transactions are not supported; finish the current transaction and start a new one instead ### Configuration @@ -86,7 +80,7 @@ These examples are with a single node ClickHouse server with ClickHouse Keeper e #### Basic configuration for a single ClickHouse server node with ClickHouse Keeper enabled :::note -See the [deployment](docs/en/deployment-guides/terminology.md) documentation for details on deploying ClickHouse server and a proper quorum of ClickHouse Keeper nodes. The configuration shown here is for experimental purposes. +See the [deployment](docs/en/deployment-guides/terminology.md) documentation for details on deploying ClickHouse server and a proper quorum of ClickHouse Keeper nodes. The configuration shown here is for experimental purposes. ::: ```xml title=/etc/clickhouse-server/config.d/config.xml @@ -133,19 +127,17 @@ See the [deployment](docs/en/deployment-guides/terminology.md) documentation for #### Verify that experimental transactions are enabled -Issue a `BEGIN TRANSACTION` or `START TRANSACTION` followed by a `ROLLBACK` to verify that experimental transactions are enabled, and that ClickHouse Keeper is enabled as it is used to track transactions. +Issue a `BEGIN TRANSACTION` followed by a `ROLLBACK` to verify that experimental transactions are enabled, and that ClickHouse Keeper is enabled as it is used to track transactions. ```sql BEGIN TRANSACTION ``` - ```response Ok. ``` :::tip If you see the following error, then check your configuration file to make sure that `allow_experimental_transactions` is set to `1` (or any value other than `0` or `false`). - ``` Code: 48. DB::Exception: Received from localhost:9000. DB::Exception: Transactions are not supported. @@ -153,18 +145,15 @@ DB::Exception: Transactions are not supported. ``` You can also check ClickHouse Keeper by issuing - ``` echo ruok | nc localhost 9181 ``` - ClickHouse Keeper should respond with `imok`. ::: ```sql ROLLBACK ``` - ```response Ok. ``` @@ -172,7 +161,7 @@ Ok. #### Create a table for testing :::tip -Creation of tables is not transactional. Run this DDL query outside of a transaction. +Creation of tables is not transactional. Run this DDL query outside of a transaction. ::: ```sql @@ -183,7 +172,6 @@ CREATE TABLE mergetree_table ENGINE = MergeTree ORDER BY n ``` - ```response Ok. ``` @@ -193,7 +181,6 @@ Ok. ```sql BEGIN TRANSACTION ``` - ```response Ok. ``` @@ -201,7 +188,6 @@ Ok. ```sql INSERT INTO mergetree_table FORMAT Values (10) ``` - ```response Ok. ``` @@ -210,13 +196,11 @@ Ok. SELECT * FROM mergetree_table ``` - ```response ┌──n─┐ │ 10 │ └────┘ ``` - :::note You can query the table from within a transaction and see that the row was inserted even though it has not yet been committed. ::: @@ -224,20 +208,16 @@ You can query the table from within a transaction and see that the row was inser #### Rollback the transaction, and query the table again Verify that the transaction is rolled back: - ```sql ROLLBACK ``` - ```response Ok. ``` - ```sql SELECT * FROM mergetree_table ``` - ```response Ok. @@ -249,7 +229,6 @@ Ok. ```sql BEGIN TRANSACTION ``` - ```response Ok. ``` @@ -257,7 +236,6 @@ Ok. ```sql INSERT INTO mergetree_table FORMAT Values (42) ``` - ```response Ok. ``` @@ -265,7 +243,6 @@ Ok. ```sql COMMIT ``` - ```response Ok. Elapsed: 0.002 sec. ``` @@ -274,7 +251,6 @@ Ok. Elapsed: 0.002 sec. SELECT * FROM mergetree_table ``` - ```response ┌──n─┐ │ 42 │ @@ -291,7 +267,6 @@ SELECT * FROM system.transactions FORMAT Vertical ``` - ```response Row 1: ────── @@ -305,3 +280,4 @@ state: RUNNING ## More Details See this [meta issue](https://github.com/ClickHouse/ClickHouse/issues/48794) to find much more extensive tests and to keep up to date with the progress. + From f5cf61c32e9f511933b56048f22aaf43fad67a6c Mon Sep 17 00:00:00 2001 From: Zach Naimon Date: Tue, 5 Mar 2024 14:28:48 -0500 Subject: [PATCH 0068/1165] update documentation without autoformat --- docs/en/sql-reference/transactions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/transactions.md b/docs/en/sql-reference/transactions.md index cb89a091d68..09cdc192b03 100644 --- a/docs/en/sql-reference/transactions.md +++ b/docs/en/sql-reference/transactions.md @@ -127,7 +127,7 @@ See the [deployment](docs/en/deployment-guides/terminology.md) documentation for #### Verify that experimental transactions are enabled -Issue a `BEGIN TRANSACTION` followed by a `ROLLBACK` to verify that experimental transactions are enabled, and that ClickHouse Keeper is enabled as it is used to track transactions. +Issue a `BEGIN TRANSACTION` or `START TRANSACTION` followed by a `ROLLBACK` to verify that experimental transactions are enabled, and that ClickHouse Keeper is enabled as it is used to track transactions. ```sql BEGIN TRANSACTION From b074477ffb841587c19313063c249bc4c35ef301 Mon Sep 17 00:00:00 2001 From: Zach Naimon Date: Tue, 5 Mar 2024 14:45:09 -0500 Subject: [PATCH 0069/1165] use new syntax in a test --- tests/integration/test_transactions/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_transactions/test.py b/tests/integration/test_transactions/test.py index 46660581223..584e59ba71b 100644 --- a/tests/integration/test_transactions/test.py +++ b/tests/integration/test_transactions/test.py @@ -67,8 +67,8 @@ def test_rollback_unfinished_on_restart1(start_cluster): tx(1, "insert into mt values (5, 50)") tx(1, "alter table mt update m = m+n in partition id '1' where 1") - # check that uncommitted insert will be rolled back on restart - tx(3, "begin transaction") + # check that uncommitted insert will be rolled back on restart (using `START TRANSACTION syntax`) + tx(3, "start transaction") tid5 = tx(3, "select transactionID()").strip() tx(3, "insert into mt values (6, 70)") From 7ed1be6f3c51db3bd4abb57c429c77e09b0cd8a8 Mon Sep 17 00:00:00 2001 From: Zach Naimon Date: Tue, 5 Mar 2024 14:46:39 -0500 Subject: [PATCH 0070/1165] fix test comment --- tests/integration/test_transactions/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_transactions/test.py b/tests/integration/test_transactions/test.py index 584e59ba71b..d63b7b6f545 100644 --- a/tests/integration/test_transactions/test.py +++ b/tests/integration/test_transactions/test.py @@ -67,7 +67,7 @@ def test_rollback_unfinished_on_restart1(start_cluster): tx(1, "insert into mt values (5, 50)") tx(1, "alter table mt update m = m+n in partition id '1' where 1") - # check that uncommitted insert will be rolled back on restart (using `START TRANSACTION syntax`) + # check that uncommitted insert will be rolled back on restart (using `START TRANSACTION` syntax) tx(3, "start transaction") tid5 = tx(3, "select transactionID()").strip() tx(3, "insert into mt values (6, 70)") From 53c9d4513c4b93ed79df305bb5c36c0cfb43ef79 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 6 Mar 2024 12:16:17 +0800 Subject: [PATCH 0071/1165] finish dev column array --- src/Columns/ColumnArray.cpp | 132 +++++++++++++++++++++++++++++++++--- src/Columns/ColumnArray.h | 3 + 2 files changed, 125 insertions(+), 10 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index b620da81ae8..aa0d5aa3e50 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -31,6 +31,7 @@ namespace ErrorCodes extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; extern const int LOGICAL_ERROR; extern const int TOO_LARGE_ARRAY_SIZE; + extern const int ILLEGAL_COLUMN; } /** Obtaining array as Field can be slow for large arrays and consume vast amount of memory. @@ -363,6 +364,19 @@ void ColumnArray::insertManyFromNumber(const ColumnArray & src, size_t position, memcpy(&data_ref[offset], &src_data[src_offset], src_size * sizeof(T)); } +void ColumnArray::insertManyFromConst(const ColumnConst & src, size_t position, size_t length) +{ + const ColumnArray * src_array = typeid_cast(&src.getDataColumn()); + if (!src_array) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Cannot insert from const column of type {} to column of type {}", + src.getDataColumn().getName(), + getName()); + + insertManyFromImpl(*src_array, 0, length, true); +} + void ColumnArray::insertManyFromString(const ColumnArray & src, size_t position, size_t length) { size_t src_size = src.sizeAt(position); @@ -400,12 +414,53 @@ void ColumnArray::insertManyFromString(const ColumnArray & src, size_t position, void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, size_t length) { + ColumnTuple & tuple = assert_cast(getData()); + const ColumnTuple & src_tuple = assert_cast(src.getData()); + /// Make temporary arrays for each components of Tuple. In the same way as for Nullable. + size_t tuple_size = tuple.tupleSize(); + size_t src_tuple_size = src_tuple.tupleSize(); + if (tuple_size == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty tuple"); + if (tuple_size != src_tuple_size) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Nested tuple size mismatch: {} vs {}", tuple_size, src_tuple_size); + + Columns temporary_arrays(tuple_size); + Columns src_temporary_arrays(tuple_size); + for (size_t i = 0; i < tuple_size; ++i) + { + temporary_arrays[i] = ColumnArray::create(tuple.getColumn(i).assumeMutable(), getOffsetsPtr()->assumeMutable()); + src_temporary_arrays[i] = ColumnArray::create(src_tuple.getColumn(i).assumeMutable(), src.getOffsetsPtr()->assumeMutable()); + assert_cast(*temporary_arrays[i]) + .insertManyFromImpl(assert_cast(*src_temporary_arrays[i]), position, length, false); + } + + Columns tuple_columns(tuple_size); + for (size_t i = 0; i < tuple_size; ++i) + tuple_columns[i] = assert_cast(*temporary_arrays[i]).getDataPtr(); + + getDataPtr() = ColumnTuple::create(std::move(tuple_columns)); } + void ColumnArray::insertManyFromNullable(const ColumnArray & src, size_t position, size_t length) { + ColumnNullable & nullable = assert_cast(getData()); + const ColumnNullable & src_nullable = assert_cast(src.getData()); + /// Process nested column without updating array offsets + auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()); + auto src_array_of_nested = ColumnArray(src_nullable.getNestedColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); + array_of_nested.insertManyFromImpl(src_array_of_nested, position, length, false); + + /// Process null map column without updating array offsets + auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()); + auto src_array_of_null_map = ColumnArray(src_nullable.getNullMapColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); + array_of_null_map.insertManyFromImpl(src_array_of_null_map, position, length, false); + + /// Update array data + getDataPtr() = ColumnNullable::create(array_of_nested.getDataPtr(), array_of_null_map.getDataPtr()); } + void ColumnArray::insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length) { size_t src_size = src.sizeAt(position); @@ -419,16 +474,73 @@ void ColumnArray::insertManyFromGeneric(const ColumnArray & src, size_t position void ColumnArray::insertManyFrom(const IColumn & src_, size_t position, size_t length) { - /// First fill offsets - const ColumnArray & src = assert_cast(src_); - size_t src_size = src.sizeAt(position); - auto & offsets_ref = getOffsets(); - size_t old_rows = offsets_ref.size(); - size_t new_rows = old_rows + length; - size_t old_size = offsets_ref.back(); - offsets_ref.resize(new_rows); - for (size_t i = 0, offset = old_size + src_size; i < length; ++i, offset += src_size) - offsets_ref[old_rows + i] = offset; + const ColumnConst * src_const = typeid_cast(&src_); + if (src_const) + return insertManyFromConst(*src_const, position, length); + + const ColumnArray * src_array = typeid_cast(&src_); + if (!src_array) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert from column of type {} to column of type {}", src_.getName(), getName()); + + return insertManyFromImpl(*src_array, position, length, true); +} + +void ColumnArray::insertManyFromImpl(const ColumnArray & src, size_t position, size_t length, bool update_offsets) +{ + /// First fill offsets if needed + if (update_offsets) + { + size_t src_size = src.sizeAt(position); + auto & offsets_ref = getOffsets(); + size_t old_rows = offsets_ref.size(); + size_t new_rows = old_rows + length; + size_t old_size = offsets_ref.back(); + offsets_ref.resize(new_rows); + for (size_t i = 0, offset = old_size + src_size; i < length; ++i, offset += src_size) + offsets_ref[old_rows + i] = offset; + } + + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNullable(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromTuple(src, position, length); + return insertManyFromGeneric(src, position, length); } void ColumnArray::insertDefault() diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 73d632a38b9..765f86ec552 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -215,6 +215,9 @@ private: ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const; /// Specializations for insertManyFrom + void insertManyFromConst(const ColumnConst & src, size_t position, size_t length); + void insertManyFromImpl(const ColumnArray & src, size_t position, size_t length, bool update_offsets = true); + template void insertManyFromNumber(const ColumnArray & src, size_t position, size_t length); void insertManyFromString(const ColumnArray & src, size_t position, size_t length); From 3bf3c7cc708d1a564896d649a1a804b868f89d8d Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 6 Mar 2024 12:32:23 +0800 Subject: [PATCH 0072/1165] finish column map and tuple --- src/Columns/ColumnArray.cpp | 2 +- src/Columns/ColumnMap.cpp | 5 +++++ src/Columns/ColumnMap.h | 1 + src/Columns/ColumnTuple.cpp | 12 ++++++++++++ src/Columns/ColumnTuple.h | 1 + 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index aa0d5aa3e50..5b0df8e9b6b 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -364,7 +364,7 @@ void ColumnArray::insertManyFromNumber(const ColumnArray & src, size_t position, memcpy(&data_ref[offset], &src_data[src_offset], src_size * sizeof(T)); } -void ColumnArray::insertManyFromConst(const ColumnConst & src, size_t position, size_t length) +void ColumnArray::insertManyFromConst(const ColumnConst & src, size_t /*position*/, size_t length) { const ColumnArray * src_array = typeid_cast(&src.getDataColumn()); if (!src_array) diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index 995f3103484..57e8ba685b4 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -158,6 +158,11 @@ void ColumnMap::insertFrom(const IColumn & src, size_t n) nested->insertFrom(assert_cast(src).getNestedColumn(), n); } +void ColumnMap::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + assert_cast(*nested).insertManyFrom(assert_cast(src).getNestedColumn(), position, length); +} + void ColumnMap::insertRangeFrom(const IColumn & src, size_t start, size_t length) { nested->insertRangeFrom( diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index 17cd86a3788..60aa69e7bf6 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -67,6 +67,7 @@ public: void updateWeakHash32(WeakHash32 & hash) const override; void updateHashFast(SipHash & hash) const override; void insertFrom(const IColumn & src_, size_t n) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; void expand(const Filter & mask, bool inverted) override; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 17cc58d92f5..062bdadf9d2 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -185,6 +185,18 @@ void ColumnTuple::insertFrom(const IColumn & src_, size_t n) columns[i]->insertFrom(*src.columns[i], n); } +void ColumnTuple::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + const ColumnTuple & src_tuple = assert_cast(src); + + const size_t tuple_size = columns.size(); + if (src_tuple.columns.size() != tuple_size) + throw Exception(ErrorCodes::CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE, "Cannot insert value of different size into tuple"); + + for (size_t i = 0; i < tuple_size; ++i) + columns[i]->insertManyFrom(*src_tuple.columns[i], position, length); +} + void ColumnTuple::insertDefault() { for (auto & column : columns) diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index 610416b8b11..5b626155754 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -60,6 +60,7 @@ public: void insert(const Field & x) override; bool tryInsert(const Field & x) override; void insertFrom(const IColumn & src_, size_t n) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertDefault() override; void popBack(size_t n) override; StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override; From 3005bff23100539dbb71f9623dc3aed9c34a87f6 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 6 Mar 2024 14:43:33 +0800 Subject: [PATCH 0073/1165] fix building --- src/Columns/ColumnArray.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 5b0df8e9b6b..389b3e97820 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -425,7 +425,7 @@ void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, if (tuple_size != src_tuple_size) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Nested tuple size mismatch: {} vs {}", tuple_size, src_tuple_size); - Columns temporary_arrays(tuple_size); + MutableColumns temporary_arrays(tuple_size); Columns src_temporary_arrays(tuple_size); for (size_t i = 0; i < tuple_size; ++i) { From 3dbb0a12fb433b29107d449099efbc99f5d71f34 Mon Sep 17 00:00:00 2001 From: unashi Date: Wed, 6 Mar 2024 16:15:37 +0800 Subject: [PATCH 0074/1165] [fix] style --- src/Storages/MergeTree/MergeTreeData.cpp | 7 +++---- tests/integration/helpers/cluster.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index d8680958c21..c76ffeee874 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7161,11 +7161,10 @@ std::pair MergeTreeData::cloneAn try { auto reservation_space = src_part_storage->reserve(src_part->getBytesOnDisk()); - if (!reservation_space) { + if (!reservation_space) throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Not enough space on disk."); - } - dst_part_storage - = src_part_storage->clonePart(this->getRelativeDataPath(), tmp_dst_part_name, disk, read_settings, write_settings, {}, {}); + dst_part_storage = src_part_storage->clonePart( + this->getRelativeDataPath(), tmp_dst_part_name, disk, read_settings, write_settings, {}, {}); copy_successful = true; break; } diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 1d96563251b..767ba5b6660 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -3216,7 +3216,7 @@ services: - timeout:1 - inet6 - rotate - {networks} + {123} {app_net} {ipv4_address} {ipv6_address} From b4dba828a4dcde93944e05b512818827fd3e5a85 Mon Sep 17 00:00:00 2001 From: unashi Date: Wed, 6 Mar 2024 16:19:07 +0800 Subject: [PATCH 0075/1165] [fix] --- tests/integration/helpers/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 767ba5b6660..1d96563251b 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -3216,7 +3216,7 @@ services: - timeout:1 - inet6 - rotate - {123} + {networks} {app_net} {ipv4_address} {ipv6_address} From 8e413da8f156ab03c875b9525044265cffcc5b83 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 6 Mar 2024 17:32:08 +0800 Subject: [PATCH 0076/1165] apply opts for string nested in array --- src/Columns/ColumnArray.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 389b3e97820..44b17c89ae1 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -536,6 +536,8 @@ void ColumnArray::insertManyFromImpl(const ColumnArray & src, size_t position, s return insertManyFromNumber(src, position, length); if (typeid_cast *>(data.get())) return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromString(src, position, length); if (typeid_cast(data.get())) return insertManyFromNullable(src, position, length); if (typeid_cast(data.get())) From 076482e8bd503ca352695173d87e9b48228389f0 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 6 Mar 2024 14:04:09 +0100 Subject: [PATCH 0077/1165] Remove whitespaces --- src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 42c6e63da01..e0d0fda81cb 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -132,7 +132,7 @@ class OptimizeShardingKeyRewriteIn : public InDepthQueryTreeVisitorWithContext; - + OptimizeShardingKeyRewriteIn(OptimizeShardingKeyRewriteInVisitor::Data data_, ContextPtr context) : Base(std::move(context)) , data(std::move(data_)) From 7065e650e1d007be4659ddb1f070b48e19cdef55 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 6 Mar 2024 14:34:03 +0100 Subject: [PATCH 0078/1165] Code cleanup --- src/DataTypes/ObjectUtils.cpp | 24 ++------ src/DataTypes/ObjectUtils.h | 2 - .../ClusterProxy/SelectStreamFactory.cpp | 7 +-- .../ClusterProxy/SelectStreamFactory.h | 2 +- src/Processors/QueryPlan/ReadFromRemote.cpp | 61 +++---------------- src/Storages/StorageDistributed.cpp | 1 - tests/analyzer_integration_broken_tests.txt | 1 - .../test_distributed_type_object/test.py | 3 +- 8 files changed, 18 insertions(+), 83 deletions(-) diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 47d8c5c9113..ccfa0a28f13 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -1,4 +1,8 @@ #include +#include +#include +#include +#include #include #include #include @@ -21,16 +25,6 @@ #include #include #include -#include "Analyzer/ConstantNode.h" -#include "Analyzer/FunctionNode.h" -#include "Analyzer/IQueryTreeNode.h" -#include "Analyzer/Identifier.h" -#include "Analyzer/IdentifierNode.h" -#include "Analyzer/QueryNode.h" -#include "Analyzer/Utils.h" -#include -#include -#include "Common/logger_useful.h" namespace DB @@ -991,22 +985,12 @@ MissingObjectList replaceMissedSubcolumnsByConstants( { auto constant = std::make_shared(type->getDefault(), type); constant->setAlias(table_expression->getAlias() + "." + name); - // auto materialize = std::make_shared("materialize"); - - // auto function = FunctionFactory::instance().get("materialize", context); - // materialize->getArguments().getNodes() = { constant }; - // materialize->resolveAsFunction(function->build(materialize->getArgumentColumns())); - // materialize->setAlias(name); column_name_to_node[name] = buildCastFunction(constant, type, context); missed_list.push_back({ constant->getValueStringRepresentation() + "_" + constant->getResultType()->getName(), table_expression->getAlias() + "." + name }); - LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "{} -> {}", missed_list.back().first, missed_list.back().second); - LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Name {} Expression\n{}", name, column_name_to_node[name]->dumpTree()); } - LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Table expression\n{} ", table_expression->dumpTree()); replaceColumns(query, table_expression, column_name_to_node); - LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Result:\n{} ", query->dumpTree()); return missed_list; } diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 013e525832e..6ef19baf5ae 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -3,8 +3,6 @@ #include #include #include -#include "Analyzer/IQueryTreeNode.h" -#include "Interpreters/Context_fwd.h" #include #include #include diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 5bcd1ce68cb..4fccd83c8c0 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -5,10 +5,9 @@ #include #include #include -#include "Analyzer/IQueryTreeNode.h" -#include "Interpreters/InterpreterSelectQueryAnalyzer.h" -#include "Interpreters/SelectQueryOptions.h" -#include "Planner/Utils.h" +#include +#include +#include #include #include #include diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index bee7edb3c19..61694830b3d 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -8,7 +9,6 @@ #include #include #include -#include "Analyzer/IQueryTreeNode.h" namespace DB { diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index ac507c6d555..72848a37f6e 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -1,4 +1,3 @@ -#include #include #include @@ -15,7 +14,6 @@ #include #include #include -#include "DataTypes/ObjectUtils.h" #include #include #include @@ -33,54 +31,14 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -static void addRenamingActions(Pipe & pipe, const MissingObjectList & missed_list, const Block & output_header) -{ - if (missed_list.empty()) - return; - - const auto & output_columns = output_header.getColumnsWithTypeAndName(); - std::vector indexes; - for (size_t i = 0; i < output_columns.size(); ++i) - { - bool found = false; - for (auto const & elem : missed_list) - { - if (output_columns[i].name.contains(elem.second)) - { - found = true; - break; - } - } - if (found) - indexes.push_back(i); - } - - auto dag = std::make_shared(pipe.getHeader().getColumnsWithTypeAndName()); - - for (size_t index : indexes) - { - dag->addOrReplaceInOutputs(dag->addAlias(*dag->getOutputs()[index], output_header.getByPosition(index).name)); - } - - // dag->addAliases(rename_to_apply); - - auto convert_actions = std::make_shared(dag); - pipe.addSimpleTransform([&](const Block & cur_header, Pipe::StreamType) -> ProcessorPtr - { - return std::make_shared(cur_header, convert_actions); - }); - - LOG_DEBUG(&Poco::Logger::get("addRenamingActions"), "EXPECTED:\n{}", output_header.dumpStructure()); - - LOG_DEBUG(&Poco::Logger::get("addRenamingActions"), "{}", pipe.getHeader().dumpStructure()); -} - -static void addConvertingActions(Pipe & pipe, const Block & header) +static void addConvertingActions(Pipe & pipe, const Block & header, bool use_positions_to_match = false) { if (blocksHaveEqualStructure(pipe.getHeader(), header)) return; - auto get_converting_dag = [](const Block & block_, const Block & header_) + auto match_mode = use_positions_to_match ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; + + auto get_converting_dag = [mode = match_mode](const Block & block_, const Block & header_) { /// Convert header structure to expected. /// Also we ignore constants from result and replace it with constants from header. @@ -88,7 +46,7 @@ static void addConvertingActions(Pipe & pipe, const Block & header) return ActionsDAG::makeConvertingActions( block_.getColumnsWithTypeAndName(), header_.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name, + mode, true); }; @@ -260,8 +218,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); - addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); } void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard) @@ -342,8 +299,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); } } else @@ -372,8 +328,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); } } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 34ab21a4751..726f1788115 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -30,7 +30,6 @@ #include #include #include -#include "Analyzer/IQueryTreeNode.h" #include #include diff --git a/tests/analyzer_integration_broken_tests.txt b/tests/analyzer_integration_broken_tests.txt index 796ca6bca22..a7954f91efa 100644 --- a/tests/analyzer_integration_broken_tests.txt +++ b/tests/analyzer_integration_broken_tests.txt @@ -1,6 +1,5 @@ test_build_sets_from_multiple_threads/test.py::test_set test_concurrent_backups_s3/test.py::test_concurrent_backups -test_distributed_type_object/test.py::test_distributed_type_object test_merge_table_over_distributed/test.py::test_global_in test_merge_table_over_distributed/test.py::test_select_table_name_from_merge_over_distributed test_passing_max_partitions_to_read_remotely/test.py::test_default_database_on_cluster diff --git a/tests/integration/test_distributed_type_object/test.py b/tests/integration/test_distributed_type_object/test.py index 7e6c000cb8e..360087c9dda 100644 --- a/tests/integration/test_distributed_type_object/test.py +++ b/tests/integration/test_distributed_type_object/test.py @@ -85,10 +85,11 @@ def test_distributed_type_object(started_cluster): 3\t\t\t\tfoo""" ) + # The following query is not supported by analyzer now assert ( TSV( node1.query( - "SELECT id, data.k1, data.k2.k3, data.k2.k4, data.k5 FROM dist_table ORDER BY id" + "SELECT id, data.k1, data.k2.k3, data.k2.k4, data.k5 FROM dist_table ORDER BY id SETTINGS allow_experimental_analyzer = 0" ) ) == expected From d9b5f9a086d8bc1af5387edee77d0e3fdbf4d9b0 Mon Sep 17 00:00:00 2001 From: Nikolay Monkov Date: Wed, 6 Mar 2024 21:37:56 +0500 Subject: [PATCH 0079/1165] version has been added to docker labels --- tests/ci/docker_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 38d0ea6d86b..803dbfcd92a 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -216,11 +216,12 @@ def gen_tags(version: ClickHouseVersion, release_type: str) -> List[str]: return tags -def buildx_args(urls: Dict[str, str], arch: str, direct_urls: List[str]) -> List[str]: +def buildx_args(urls: Dict[str, str], arch: str, direct_urls: List[str], version: str) -> List[str]: args = [ f"--platform=linux/{arch}", f"--label=build-url={GITHUB_RUN_URL}", f"--label=com.clickhouse.build.githash={git.sha}", + f"--label=com.clickhouse.build.version={version}", ] if direct_urls: args.append(f"--build-arg=DIRECT_DOWNLOAD_URLS='{' '.join(direct_urls)}'") @@ -267,7 +268,7 @@ def build_and_push_image( urls = [url for url in direct_urls[arch] if ".deb" in url] else: urls = [url for url in direct_urls[arch] if ".tgz" in url] - cmd_args.extend(buildx_args(repo_urls, arch, direct_urls=urls)) + cmd_args.extend(buildx_args(repo_urls, arch, direct_urls=urls, version=version.describe)) if not push: cmd_args.append(f"--tag={image.repo}:{arch_tag}") cmd_args.extend( From fb17749b50ce1024ef8c0b6f7bb8b7a58321894c Mon Sep 17 00:00:00 2001 From: Nikolay Monkov Date: Thu, 7 Mar 2024 09:45:24 +0500 Subject: [PATCH 0080/1165] file has been reformatted to pass Style check --- tests/ci/docker_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 803dbfcd92a..35c86d8eadd 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -216,7 +216,9 @@ def gen_tags(version: ClickHouseVersion, release_type: str) -> List[str]: return tags -def buildx_args(urls: Dict[str, str], arch: str, direct_urls: List[str], version: str) -> List[str]: +def buildx_args( + urls: Dict[str, str], arch: str, direct_urls: List[str], version: str +) -> List[str]: args = [ f"--platform=linux/{arch}", f"--label=build-url={GITHUB_RUN_URL}", @@ -268,7 +270,9 @@ def build_and_push_image( urls = [url for url in direct_urls[arch] if ".deb" in url] else: urls = [url for url in direct_urls[arch] if ".tgz" in url] - cmd_args.extend(buildx_args(repo_urls, arch, direct_urls=urls, version=version.describe)) + cmd_args.extend( + buildx_args(repo_urls, arch, direct_urls=urls, version=version.describe) + ) if not push: cmd_args.append(f"--tag={image.repo}:{arch_tag}") cmd_args.extend( From 10b5ce8ab3d1b412f6500d03bc96e205965178d7 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 7 Mar 2024 10:26:50 +0100 Subject: [PATCH 0081/1165] Updated BufferAllocationPolicy --- src/Backups/BackupIO_AzureBlobStorage.cpp | 4 +-- .../BufferAllocationPolicy.cpp} | 25 +++++-------- src/Common/BufferAllocationPolicy.h | 35 +++++++++++++++++++ src/Common/ThreadPoolTaskTracker.h | 2 -- src/Core/Settings.h | 5 +++ .../IO/WriteBufferFromAzureBlobStorage.cpp | 23 +++++++----- .../IO/WriteBufferFromAzureBlobStorage.h | 9 ++--- .../AzureBlobStorage/AzureBlobStorageAuth.cpp | 29 ++++++++------- .../AzureBlobStorage/AzureObjectStorage.cpp | 4 +-- .../AzureBlobStorage/AzureObjectStorage.h | 17 +++++++-- src/IO/WriteBufferFromS3.cpp | 7 +++- src/IO/WriteBufferFromS3.h | 13 +------ 12 files changed, 107 insertions(+), 66 deletions(-) rename src/{IO/WriteBufferFromS3BufferAllocationPolicy.cpp => Common/BufferAllocationPolicy.cpp} (74%) create mode 100644 src/Common/BufferAllocationPolicy.h diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index fb36248433d..8d2b217ad21 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -275,11 +275,9 @@ std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const Strin return std::make_unique( client, key, - settings->max_single_part_upload_size, - settings->max_unexpected_write_error_retries, DBMS_DEFAULT_BUFFER_SIZE, write_settings, - settings->max_inflight_parts_for_one_file); + settings); } void BackupWriterAzureBlobStorage::removeFile(const String & file_name) diff --git a/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp b/src/Common/BufferAllocationPolicy.cpp similarity index 74% rename from src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp rename to src/Common/BufferAllocationPolicy.cpp index 6347c1acfd7..1456233eb03 100644 --- a/src/IO/WriteBufferFromS3BufferAllocationPolicy.cpp +++ b/src/Common/BufferAllocationPolicy.cpp @@ -1,21 +1,17 @@ -#include "config.h" - -#if USE_AWS_S3 - -#include +#include "BufferAllocationPolicy.h" #include -namespace +namespace DB { -class FixedSizeBufferAllocationPolicy : public DB::WriteBufferFromS3::IBufferAllocationPolicy +class FixedSizeBufferAllocationPolicy : public IBufferAllocationPolicy { const size_t buffer_size = 0; size_t buffer_number = 0; public: - explicit FixedSizeBufferAllocationPolicy(const DB::S3Settings::RequestSettings::PartUploadSettings & settings_) + explicit FixedSizeBufferAllocationPolicy(const BufferAllocationSettings & settings_) : buffer_size(settings_.strict_upload_part_size) { chassert(buffer_size > 0); @@ -36,7 +32,7 @@ public: }; -class ExpBufferAllocationPolicy : public DB::WriteBufferFromS3::IBufferAllocationPolicy +class ExpBufferAllocationPolicy : public DB::IBufferAllocationPolicy { const size_t first_size = 0; const size_t second_size = 0; @@ -49,7 +45,7 @@ class ExpBufferAllocationPolicy : public DB::WriteBufferFromS3::IBufferAllocatio size_t buffer_number = 0; public: - explicit ExpBufferAllocationPolicy(const DB::S3Settings::RequestSettings::PartUploadSettings & settings_) + explicit ExpBufferAllocationPolicy(const BufferAllocationSettings & settings_) : first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size)) , second_size(settings_.min_upload_part_size) , multiply_factor(settings_.upload_part_size_multiply_factor) @@ -92,14 +88,10 @@ public: } }; -} -namespace DB -{ +IBufferAllocationPolicy::~IBufferAllocationPolicy() = default; -WriteBufferFromS3::IBufferAllocationPolicy::~IBufferAllocationPolicy() = default; - -WriteBufferFromS3::IBufferAllocationPolicyPtr WriteBufferFromS3::ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_) +IBufferAllocationPolicyPtr ChooseBufferPolicy(BufferAllocationSettings settings_) { if (settings_.strict_upload_part_size > 0) return std::make_unique(settings_); @@ -109,4 +101,3 @@ WriteBufferFromS3::IBufferAllocationPolicyPtr WriteBufferFromS3::ChooseBufferPol } -#endif diff --git a/src/Common/BufferAllocationPolicy.h b/src/Common/BufferAllocationPolicy.h new file mode 100644 index 00000000000..b759d22ede6 --- /dev/null +++ b/src/Common/BufferAllocationPolicy.h @@ -0,0 +1,35 @@ +#pragma once + +#include "config.h" + +#include "logger_useful.h" + +#include + +namespace DB +{ + +struct BufferAllocationSettings +{ + size_t strict_upload_part_size = 0; + size_t min_upload_part_size = 16 * 1024 * 1024; + size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; + size_t upload_part_size_multiply_factor = 2; + size_t upload_part_size_multiply_parts_count_threshold = 500; + size_t max_single_part_upload_size = 32 * 1024 * 1024; +}; + +class IBufferAllocationPolicy +{ + public: + virtual size_t getBufferNumber() const = 0; + virtual size_t getBufferSize() const = 0; + virtual void nextBuffer() = 0; + virtual ~IBufferAllocationPolicy() = 0; +}; + +using IBufferAllocationPolicyPtr = std::unique_ptr; + +IBufferAllocationPolicyPtr ChooseBufferPolicy(BufferAllocationSettings settings_); + +} diff --git a/src/Common/ThreadPoolTaskTracker.h b/src/Common/ThreadPoolTaskTracker.h index d37b759a913..72591648d30 100644 --- a/src/Common/ThreadPoolTaskTracker.h +++ b/src/Common/ThreadPoolTaskTracker.h @@ -11,8 +11,6 @@ namespace DB { -/// That class is used only in WriteBufferFromS3 for now. -/// Therefore it declared as a part of WriteBufferFromS3. /// TaskTracker takes a Callback which is run by scheduler in some external shared ThreadPool. /// TaskTracker brings the methods waitIfAny, waitAll/safeWaitAll /// to help with coordination of the running tasks. diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b186ca6fe01..8ad08b7e348 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -78,10 +78,15 @@ class IColumn; M(UInt64, distributed_connections_pool_size, 1024, "Maximum number of connections with one remote server in the pool.", 0) \ M(UInt64, connections_with_failover_max_tries, 3, "The maximum number of attempts to connect to replicas.", 0) \ M(UInt64, s3_strict_upload_part_size, 0, "The exact size of part to upload during multipart upload to S3 (some implementations does not supports variable size parts).", 0) \ + M(UInt64, azure_strict_upload_part_size, 0, "The exact size of part to upload during multipart upload to Azure blob storage.", 0) \ M(UInt64, s3_min_upload_part_size, 16*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \ M(UInt64, s3_max_upload_part_size, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to S3.", 0) \ + M(UInt64, azure_min_upload_part_size, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage.", 0) \ + M(UInt64, azure_max_upload_part_size, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage.", 0) \ M(UInt64, s3_upload_part_size_multiply_factor, 2, "Multiply s3_min_upload_part_size by this factor each time s3_multiply_parts_count_threshold parts were uploaded from a single write to S3.", 0) \ M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, 500, "Each time this number of parts was uploaded to S3, s3_min_upload_part_size is multiplied by s3_upload_part_size_multiply_factor.", 0) \ + M(UInt64, azure_upload_part_size_multiply_factor, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage.", 0) \ + M(UInt64, azure_upload_part_size_multiply_parts_count_threshold, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor.", 0) \ M(UInt64, s3_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \ M(UInt64, azure_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \ M(UInt64, s3_max_single_part_upload_size, 32*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \ diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index 74a8949b235..bc11d445a51 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -27,23 +27,27 @@ struct WriteBufferFromAzureBlobStorage::PartData WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( std::shared_ptr blob_container_client_, const String & blob_path_, - size_t max_single_part_upload_size_, - size_t max_unexpected_write_error_retries_, size_t buf_size_, const WriteSettings & write_settings_, - size_t max_inflight_parts_for_one_file_, + std::shared_ptr settings_, ThreadPoolCallbackRunner schedule_) : WriteBufferFromFileBase(buf_size_, nullptr, 0) , log(getLogger("WriteBufferFromAzureBlobStorage")) - , max_single_part_upload_size(max_single_part_upload_size_) - , max_unexpected_write_error_retries(max_unexpected_write_error_retries_) + , buffer_allocation_policy(ChooseBufferPolicy({settings_->strict_upload_part_size, + settings_->min_upload_part_size, + settings_->max_upload_part_size, + settings_->upload_part_size_multiply_factor, + settings_->upload_part_size_multiply_parts_count_threshold, + settings_->max_single_part_upload_size})) + , max_single_part_upload_size(settings_->max_single_part_upload_size) + , max_unexpected_write_error_retries(settings_->max_unexpected_write_error_retries) , blob_path(blob_path_) , write_settings(write_settings_) , blob_container_client(blob_container_client_) , task_tracker( std::make_unique( std::move(schedule_), - max_inflight_parts_for_one_file_, + settings_->max_inflight_parts_for_one_file, limitedLog)) { allocateBuffer(); @@ -119,7 +123,8 @@ void WriteBufferFromAzureBlobStorage::nextImpl() void WriteBufferFromAzureBlobStorage::allocateBuffer() { - memory = Memory(max_single_part_upload_size); + buffer_allocation_policy->nextBuffer(); + memory = Memory(buffer_allocation_policy->getBufferSize()); WriteBuffer::set(memory.data(), memory.size()); } @@ -129,10 +134,10 @@ void WriteBufferFromAzureBlobStorage::reallocateBuffer() if (available() > 0) return; - if (memory.size() == max_single_part_upload_size) + if (memory.size() == buffer_allocation_policy->getBufferSize()) return; - memory.resize(max_single_part_upload_size); + memory.resize(buffer_allocation_policy->getBufferSize()); WriteBuffer::set(memory.data(), memory.size()); diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index 4897ca9a846..7223f66693e 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -12,7 +12,8 @@ #include #include #include - +#include +#include namespace Poco { @@ -32,11 +33,9 @@ public: WriteBufferFromAzureBlobStorage( AzureClientPtr blob_container_client_, const String & blob_path_, - size_t max_single_part_upload_size_, - size_t max_unexpected_write_error_retries_, size_t buf_size_, const WriteSettings & write_settings_, - size_t max_inflight_parts_for_one_file_, + std::shared_ptr settings_, ThreadPoolCallbackRunner schedule_ = {}); ~WriteBufferFromAzureBlobStorage() override; @@ -63,6 +62,8 @@ private: LoggerPtr log; LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); + IBufferAllocationPolicyPtr buffer_allocation_policy; + const size_t max_single_part_upload_size; const size_t max_unexpected_write_error_retries; const std::string blob_path; diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index f99586b2d1a..1d01e2f45e3 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -160,18 +160,23 @@ std::unique_ptr getAzureBlobContainerClient( std::unique_ptr getAzureBlobStorageSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context) { - return std::make_unique( - config.getUInt64(config_prefix + ".max_single_part_upload_size", 100 * 1024 * 1024), - config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024), - config.getInt(config_prefix + ".max_single_read_retries", 3), - config.getInt(config_prefix + ".max_single_download_retries", 3), - config.getInt(config_prefix + ".list_object_keys_size", 1000), - config.getUInt64(config_prefix + ".max_upload_part_size", 5ULL * 1024 * 1024 * 1024), - config.getUInt64(config_prefix + ".max_single_part_copy_size", context->getSettings().azure_max_single_part_copy_size), - config.getBool(config_prefix + ".use_native_copy", false), - config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", context->getSettings().azure_max_unexpected_write_error_retries), - config.getUInt64(config_prefix + ".max_inflight_parts_for_one_file", context->getSettings().azure_max_inflight_parts_for_one_file) - ); + std::unique_ptr settings = std::make_unique(); + settings->max_single_part_upload_size = config.getUInt64(config_prefix + ".max_single_part_upload_size", 100 * 1024 * 1024); + settings->min_bytes_for_seek = config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024); + settings->max_single_read_retries = config.getInt(config_prefix + ".max_single_read_retries", 3); + settings->max_single_download_retries = config.getInt(config_prefix + ".max_single_download_retries", 3); + settings->list_object_keys_size = config.getInt(config_prefix + ".list_object_keys_size", 1000); + settings->min_upload_part_size = config.getUInt64(config_prefix + ".min_upload_part_size", context->getSettings().azure_min_upload_part_size); + settings->max_upload_part_size = config.getUInt64(config_prefix + ".max_upload_part_size", context->getSettings().azure_max_upload_part_size); + settings->max_single_part_copy_size = config.getUInt64(config_prefix + ".max_single_part_copy_size", context->getSettings().azure_max_single_part_copy_size); + settings->use_native_copy = config.getBool(config_prefix + ".use_native_copy", false); + settings->max_unexpected_write_error_retries = config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", context->getSettings().azure_max_unexpected_write_error_retries); + settings->max_inflight_parts_for_one_file = config.getUInt64(config_prefix + ".max_inflight_parts_for_one_file", context->getSettings().azure_max_inflight_parts_for_one_file); + settings->strict_upload_part_size = config.getUInt64(config_prefix + ".strict_upload_part_size", context->getSettings().azure_strict_upload_part_size); + settings->upload_part_size_multiply_factor = config.getUInt64(config_prefix + ".upload_part_size_multiply_factor", context->getSettings().azure_upload_part_size_multiply_factor); + settings->upload_part_size_multiply_parts_count_threshold = config.getUInt64(config_prefix + ".upload_part_size_multiply_parts_count_threshold", context->getSettings().azure_upload_part_size_multiply_parts_count_threshold); + + return settings; } } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index 844789ea5b5..15ab55d5611 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -265,11 +265,9 @@ std::unique_ptr AzureObjectStorage::writeObject( /// NO return std::make_unique( client.get(), object.remote_path, - settings.get()->max_single_part_upload_size, - settings.get()->max_unexpected_write_error_retries, buf_size, patchSettings(write_settings), - settings.get()->max_inflight_parts_for_one_file); + settings.get()); } /// Remove file. Throws exception if file doesn't exists or it's a directory. diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 1b473a01304..b97d706a4d9 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -24,21 +24,29 @@ struct AzureObjectStorageSettings int max_single_read_retries_, int max_single_download_retries_, int list_object_keys_size_, + size_t min_upload_part_size_, size_t max_upload_part_size_, size_t max_single_part_copy_size_, bool use_native_copy_, size_t max_unexpected_write_error_retries_, - size_t max_inflight_parts_for_one_file_) + size_t max_inflight_parts_for_one_file_, + size_t strict_upload_part_size_, + size_t upload_part_size_multiply_factor_, + size_t upload_part_size_multiply_parts_count_threshold_) : max_single_part_upload_size(max_single_part_upload_size_) , min_bytes_for_seek(min_bytes_for_seek_) , max_single_read_retries(max_single_read_retries_) , max_single_download_retries(max_single_download_retries_) , list_object_keys_size(list_object_keys_size_) + , min_upload_part_size(min_upload_part_size_) , max_upload_part_size(max_upload_part_size_) , max_single_part_copy_size(max_single_part_copy_size_) , use_native_copy(use_native_copy_) - , max_unexpected_write_error_retries (max_unexpected_write_error_retries_) - , max_inflight_parts_for_one_file (max_inflight_parts_for_one_file_) + , max_unexpected_write_error_retries(max_unexpected_write_error_retries_) + , max_inflight_parts_for_one_file(max_inflight_parts_for_one_file_) + , strict_upload_part_size(strict_upload_part_size_) + , upload_part_size_multiply_factor(upload_part_size_multiply_factor_) + , upload_part_size_multiply_parts_count_threshold(upload_part_size_multiply_parts_count_threshold_) { } @@ -55,6 +63,9 @@ struct AzureObjectStorageSettings bool use_native_copy = false; size_t max_unexpected_write_error_retries = 4; size_t max_inflight_parts_for_one_file = 20; + size_t strict_upload_part_size = 0; + size_t upload_part_size_multiply_factor = 2; + size_t upload_part_size_multiply_parts_count_threshold = 500; }; using AzureClient = Azure::Storage::Blobs::BlobContainerClient; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 510d9bef4d3..60fa828d6c4 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -93,7 +93,12 @@ WriteBufferFromS3::WriteBufferFromS3( , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) , object_metadata(std::move(object_metadata_)) - , buffer_allocation_policy(ChooseBufferPolicy(upload_settings)) + , buffer_allocation_policy(ChooseBufferPolicy({upload_settings.strict_upload_part_size, + upload_settings.min_upload_part_size, + upload_settings.max_upload_part_size, + upload_settings.upload_part_size_multiply_factor, + upload_settings.upload_part_size_multiply_parts_count_threshold, + upload_settings.max_single_part_upload_size})) , task_tracker( std::make_unique( std::move(schedule_), diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index afd8b9909c1..840274c8ace 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -49,18 +50,6 @@ public: std::string getFileName() const override { return key; } void sync() override { next(); } - class IBufferAllocationPolicy - { - public: - virtual size_t getBufferNumber() const = 0; - virtual size_t getBufferSize() const = 0; - virtual void nextBuffer() = 0; - virtual ~IBufferAllocationPolicy() = 0; - }; - using IBufferAllocationPolicyPtr = std::unique_ptr; - - static IBufferAllocationPolicyPtr ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_); - private: /// Receives response from the server after sending all data. void finalizeImpl() override; From 930deee699be05398aac334ce9e025d084c68a30 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Thu, 7 Mar 2024 22:02:10 +0800 Subject: [PATCH 0082/1165] fix bugs --- src/Columns/ColumnArray.cpp | 63 ++++++++++++++++++------------------- src/Columns/ColumnArray.h | 2 +- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 44b17c89ae1..0214375122f 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -43,29 +43,34 @@ namespace ErrorCodes static constexpr size_t max_array_size_as_field = 1000000; -ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column) +ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column, bool check_offsets) : data(std::move(nested_column)), offsets(std::move(offsets_column)) { - const ColumnOffsets * offsets_concrete = typeid_cast(offsets.get()); - - if (!offsets_concrete) - throw Exception(ErrorCodes::LOGICAL_ERROR, "offsets_column must be a ColumnUInt64"); - - if (!offsets_concrete->empty() && data && !data->empty()) + if (check_offsets) { - Offset last_offset = offsets_concrete->getData().back(); + const ColumnOffsets * offsets_concrete = typeid_cast(offsets.get()); - /// This will also prevent possible overflow in offset. - if (data->size() != last_offset) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "offsets_column has data inconsistent with nested_column. Data size: {}, last offset: {}", - data->size(), last_offset); + if (!offsets_concrete) + throw Exception(ErrorCodes::LOGICAL_ERROR, "offsets_column must be a ColumnUInt64"); + + if (!offsets_concrete->empty() && data && !data->empty()) + { + Offset last_offset = offsets_concrete->getData().back(); + + /// This will also prevent possible overflow in offset. + if (data->size() != last_offset) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "offsets_column has data inconsistent with nested_column. Data size: {}, last offset: {}", + data->size(), + last_offset); + } + + /** NOTE + * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). + * But in most cases, arrays with constant value are unexpected and code will work wrong. Use with caution. + */ } - - /** NOTE - * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). - * But in most cases, arrays with constant value are unexpected and code will work wrong. Use with caution. - */ } ColumnArray::ColumnArray(MutableColumnPtr && nested_column) @@ -425,20 +430,14 @@ void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, if (tuple_size != src_tuple_size) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Nested tuple size mismatch: {} vs {}", tuple_size, src_tuple_size); - MutableColumns temporary_arrays(tuple_size); - Columns src_temporary_arrays(tuple_size); - for (size_t i = 0; i < tuple_size; ++i) - { - temporary_arrays[i] = ColumnArray::create(tuple.getColumn(i).assumeMutable(), getOffsetsPtr()->assumeMutable()); - src_temporary_arrays[i] = ColumnArray::create(src_tuple.getColumn(i).assumeMutable(), src.getOffsetsPtr()->assumeMutable()); - assert_cast(*temporary_arrays[i]) - .insertManyFromImpl(assert_cast(*src_temporary_arrays[i]), position, length, false); - } - Columns tuple_columns(tuple_size); for (size_t i = 0; i < tuple_size; ++i) - tuple_columns[i] = assert_cast(*temporary_arrays[i]).getDataPtr(); - + { + auto array_of_element = ColumnArray(tuple.getColumn(i).assumeMutable(), getOffsetsPtr()->assumeMutable(), false); + auto src_array_of_element = ColumnArray(src_tuple.getColumn(i).assumeMutable(), src.getOffsetsPtr()->assumeMutable()); + array_of_element.insertManyFromImpl(src_array_of_element, position, length, false); + tuple_columns[i] = array_of_element.getDataPtr(); + } getDataPtr() = ColumnTuple::create(std::move(tuple_columns)); } @@ -448,12 +447,12 @@ void ColumnArray::insertManyFromNullable(const ColumnArray & src, size_t positio const ColumnNullable & src_nullable = assert_cast(src.getData()); /// Process nested column without updating array offsets - auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()); + auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable(), false); auto src_array_of_nested = ColumnArray(src_nullable.getNestedColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); array_of_nested.insertManyFromImpl(src_array_of_nested, position, length, false); /// Process null map column without updating array offsets - auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()); + auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable(), false); auto src_array_of_null_map = ColumnArray(src_nullable.getNullMapColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); array_of_null_map.insertManyFromImpl(src_array_of_null_map, position, length, false); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 765f86ec552..8c4d103e7d0 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -21,7 +21,7 @@ private: friend class COWHelper, ColumnArray>; /** Create an array column with specified values and offsets. */ - ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column); + ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column, bool check_offsets = true); /** Create an empty column of arrays with the type of values as in the column `nested_column` */ explicit ColumnArray(MutableColumnPtr && nested_column); From 88d8880267d279817797fb770ba89335f258c8d0 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 7 Mar 2024 14:53:28 +0000 Subject: [PATCH 0083/1165] better virtual columns and tests added --- src/Storages/MergeTree/IMergeTreeReader.cpp | 2 +- src/Storages/MergeTree/MergeTask.cpp | 4 +- src/Storages/MergeTree/MergeTask.h | 4 +- .../MergeTree/MergeTreeBlockReadUtils.cpp | 6 +-- src/Storages/MergeTree/MergeTreeData.cpp | 4 +- .../MergeTree/MergeTreeRangeReader.cpp | 5 +- src/Storages/MergeTree/MergeTreeRangeReader.h | 3 ++ .../MergeTree/MergeTreeSequentialSource.cpp | 3 +- .../MergeTree/MergeTreeVirtualColumns.cpp | 2 +- src/Storages/MergeTree/MutateTask.cpp | 2 +- ...=> 03001_block_offset_column.reference.j2} | 2 + .../0_stateless/03001_block_offset_column.sql | 34 ------------- .../03001_block_offset_column.sql.j2 | 44 +++++++++++++++++ .../03001_block_offset_column_2.reference | 49 +++++++++++++++++++ .../03001_block_offset_column_2.sql | 25 ++++++++++ .../0_stateless/03001_data_version_column.sql | 8 +-- .../03002_sample_factor_where.reference | 3 ++ .../0_stateless/03002_sample_factor_where.sql | 11 +++++ 18 files changed, 159 insertions(+), 52 deletions(-) rename tests/queries/0_stateless/{03001_block_offset_column.reference => 03001_block_offset_column.reference.j2} (93%) delete mode 100644 tests/queries/0_stateless/03001_block_offset_column.sql create mode 100644 tests/queries/0_stateless/03001_block_offset_column.sql.j2 create mode 100644 tests/queries/0_stateless/03001_block_offset_column_2.reference create mode 100644 tests/queries/0_stateless/03001_block_offset_column_2.sql create mode 100644 tests/queries/0_stateless/03002_sample_factor_where.reference create mode 100644 tests/queries/0_stateless/03002_sample_factor_where.sql diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 41e8db8fac4..cf6b64aac85 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -95,7 +95,7 @@ void IMergeTreeReader::fillVirtualColumns(Columns & columns, size_t rows) const it->name, it->type->getName(), virtual_column->type->getName()); } - if (it->name == "_part_offset" || it->name == BlockOffsetColumn::name) + if (MergeTreeRangeReader::virtuals_to_fill.contains(it->name)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Virtual column {} must be filled by range reader", it->name); Field field; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index b61cf236435..3453e4339fa 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -224,10 +224,10 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() ctx->need_remove_expired_values = false; ctx->force_ttl = false; - if (enableBlockNumberColumn(global_ctx)) + if (enabledBlockNumberColumn(global_ctx)) addGatheringColumn(global_ctx, BlockNumberColumn::name, BlockNumberColumn::type); - if (enableBlockOffsetColumn(global_ctx)) + if (enabledBlockOffsetColumn(global_ctx)) addGatheringColumn(global_ctx, BlockOffsetColumn::name, BlockOffsetColumn::type); SerializationInfo::Settings info_settings = diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index b36fd35cff9..f6268886b14 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -404,12 +404,12 @@ private: Stages::const_iterator stages_iterator = stages.begin(); - static bool enableBlockNumberColumn(GlobalRuntimeContextPtr global_ctx) + static bool enabledBlockNumberColumn(GlobalRuntimeContextPtr global_ctx) { return global_ctx->data->getSettings()->enable_block_number_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); } - static bool enableBlockOffsetColumn(GlobalRuntimeContextPtr global_ctx) + static bool enabledBlockOffsetColumn(GlobalRuntimeContextPtr global_ctx) { return global_ctx->data->getSettings()->enable_block_offset_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); } diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index d1b2c2558ef..570387a7046 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -278,17 +278,17 @@ MergeTreeReadTaskColumns getReadTaskColumns( .withVirtuals() .withSubcolumns(with_subcolumns); - static const NameSet columns_to_read_at_first_step = {"_part_offset", BlockOffsetColumn::name}; - NameSet columns_from_previous_steps; auto add_step = [&](const PrewhereExprStep & step) { Names step_column_names; + /// Virtual columns that are filled by RangeReader + /// must be read in the first step before any filtering. if (columns_from_previous_steps.empty()) { for (const auto & required_column : required_columns) - if (columns_to_read_at_first_step.contains(required_column)) + if (MergeTreeRangeReader::virtuals_to_fill.contains(required_column)) step_column_names.push_back(required_column); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 947158b2aa1..6fb9bd07f10 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -440,7 +440,7 @@ VirtualColumnsDescription MergeTreeData::createVirtuals(const StorageInMemoryMet desc.addEphemeral("_partition_id", std::make_shared(std::make_shared()), "Name of partition"); desc.addEphemeral("_sample_factor", std::make_shared(), "Sample factor (from the query)"); desc.addEphemeral("_part_offset", std::make_shared(), "Number of row in the part"); - desc.addEphemeral("_data_version", std::make_shared(), "Data version of mutation of part"); + desc.addEphemeral("_part_data_version", std::make_shared(), "Data version of part (either min block number or mutation version)"); if (metadata.hasPartitionKey()) { @@ -1028,7 +1028,7 @@ void MergeTreeData::MergingParams::check(const StorageInMemoryMetadata & metadat /// TODO Checks for Graphite mode. } -const Names MergeTreeData::virtuals_useful_for_filter = {"_part", "_partition_id", "_part_uuid", "_partition_value", "_data_version"}; +const Names MergeTreeData::virtuals_useful_for_filter = {"_part", "_partition_id", "_part_uuid", "_partition_value", "_part_data_version"}; Block MergeTreeData::getHeaderWithVirtualsForFilter() const { diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index ec1fef20267..eb213fdc5ad 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -872,6 +872,8 @@ size_t MergeTreeRangeReader::currentMark() const return stream.currentMark(); } +const NameSet MergeTreeRangeReader::virtuals_to_fill = {"_part_offset", BlockOffsetColumn::name}; + size_t MergeTreeRangeReader::Stream::numPendingRows() const { size_t rows_between_marks = index_granularity->getRowsCountInRange(current_mark, last_mark); @@ -1144,6 +1146,7 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t if (!result.rows_per_granule.empty()) result.adjustLastGranule(); + /// Column _block_offset is the same as _part_offset if it's not persisted in part. bool has_part_offset = read_sample_block.has("_part_offset"); bool has_block_offset = read_sample_block.has(BlockOffsetColumn::name); diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.h b/src/Storages/MergeTree/MergeTreeRangeReader.h index 688a6b0922b..51fbbc8b052 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.h +++ b/src/Storages/MergeTree/MergeTreeRangeReader.h @@ -115,6 +115,9 @@ public: bool isCurrentRangeFinished() const; bool isInitialized() const { return is_initialized; } + /// Names of virtual columns that are filled in RangeReader. + static const NameSet virtuals_to_fill; + private: /// Accumulates sequential read() requests to perform a large read instead of multiple small reads class DelayedStream diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 1d29eea4c68..1dc34e0f37e 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -195,6 +195,7 @@ static void fillBlockNumberColumns( Columns & res_columns, const NamesAndTypesList & columns_list, UInt64 block_number, + UInt64 block_offset, UInt64 num_rows) { chassert(res_columns.size() == columns_list.size()); @@ -240,7 +241,7 @@ try if (rows_read) { - fillBlockNumberColumns(columns, sample, data_part->info.min_block, rows_read); + fillBlockNumberColumns(columns, sample, data_part->info.min_block, current_row, rows_read); reader->fillVirtualColumns(columns, rows_read); current_row += rows_read; diff --git a/src/Storages/MergeTree/MergeTreeVirtualColumns.cpp b/src/Storages/MergeTree/MergeTreeVirtualColumns.cpp index 0a7f0dfbdbc..821724a3cfb 100644 --- a/src/Storages/MergeTree/MergeTreeVirtualColumns.cpp +++ b/src/Storages/MergeTree/MergeTreeVirtualColumns.cpp @@ -47,7 +47,7 @@ Field getFieldForConstVirtualColumn(const String & column_name, const IMergeTree if (column_name == "_partition_id") return part.info.partition_id; - if (column_name == "_data_version") + if (column_name == "_part_data_version") return part.info.getDataVersion(); if (column_name == "_partition_value") diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index cafd4e4ed68..1a6c6c0073b 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -327,7 +327,7 @@ getColumnsForNewDataPart( } - auto persistent_virtuals = source_part->storage.getVirtualsDescription()->get(VirtualsKind::Persistent); + auto persistent_virtuals = source_part->storage.getVirtualsPtr()->getNamesAndTypesList(VirtualsKind::Persistent); for (const auto & [name, type] : persistent_virtuals) { diff --git a/tests/queries/0_stateless/03001_block_offset_column.reference b/tests/queries/0_stateless/03001_block_offset_column.reference.j2 similarity index 93% rename from tests/queries/0_stateless/03001_block_offset_column.reference rename to tests/queries/0_stateless/03001_block_offset_column.reference.j2 index fba88974380..641409167bf 100644 --- a/tests/queries/0_stateless/03001_block_offset_column.reference +++ b/tests/queries/0_stateless/03001_block_offset_column.reference.j2 @@ -1,3 +1,4 @@ +{% for enable_vertical_merge_algorithm in [0, 1] -%} *** BEFORE MUTATION BEFORE MERGE *** 1 1 1 0 all_1_1_0 2 2 2 0 all_2_2_0 @@ -39,3 +40,4 @@ 7 7 4 0 all_1_4_2_3 8 8 4 1 all_1_4_2_3 9 9 4 2 all_1_4_2_3 +{% endfor -%} diff --git a/tests/queries/0_stateless/03001_block_offset_column.sql b/tests/queries/0_stateless/03001_block_offset_column.sql deleted file mode 100644 index 0ef2b58f77b..00000000000 --- a/tests/queries/0_stateless/03001_block_offset_column.sql +++ /dev/null @@ -1,34 +0,0 @@ -DROP TABLE IF EXISTS test; - -CREATE TABLE test (id UInt32, a UInt32) ENGINE = MergeTree -ORDER BY id -SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; - -INSERT INTO test(id,a) VALUES (1,1),(3,3),(4,4); -INSERT INTO test(id,a) VALUES (2,2),(5,5),(6,6); - -SELECT '*** BEFORE MUTATION BEFORE MERGE ***'; -SELECT id,a,_block_number,_block_offset,_part from test ORDER BY id; - -set mutations_sync=1; -ALTER TABLE test UPDATE a=0 WHERE id<4; - -SELECT '*** AFTER MUTATION BEFORE MERGE ***'; -SELECT id,a,_block_number,_block_offset,_part from test ORDER BY id; - -OPTIMIZE TABLE test FINAL; - -SELECT '*** AFTER MUTATION AFTER MERGE ***'; -SELECT *,_block_number,_block_offset,_part from test ORDER BY id; - -INSERT INTO test(id,a) VALUES (7,7),(8,8),(9,9); - -SELECT '*** AFTER MUTATION AFTER MERGE , NEW BLOCK ***'; -SELECT *,_block_number,_block_offset,_part from test ORDER BY id; - -OPTIMIZE TABLE test FINAL; - -SELECT '*** AFTER MUTATION AFTER MERGE , NEW BLOCK MERGED ***'; -SELECT *,_block_number,_block_offset,_part from test ORDER BY id; - -DROP TABLE test; \ No newline at end of file diff --git a/tests/queries/0_stateless/03001_block_offset_column.sql.j2 b/tests/queries/0_stateless/03001_block_offset_column.sql.j2 new file mode 100644 index 00000000000..b161183f884 --- /dev/null +++ b/tests/queries/0_stateless/03001_block_offset_column.sql.j2 @@ -0,0 +1,44 @@ +{% for enable_vertical_merge_algorithm in [0, 1] -%} + +DROP TABLE IF EXISTS t_block_offset; + +CREATE TABLE t_block_offset (id UInt32, a UInt32) ENGINE = MergeTree +ORDER BY id +SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + index_granularity = 2, + vertical_merge_algorithm_min_bytes_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + enable_vertical_merge_algorithm = {{ enable_vertical_merge_algorithm }}; + +INSERT INTO t_block_offset(id,a) VALUES (1,1),(3,3),(4,4); +INSERT INTO t_block_offset(id,a) VALUES (2,2),(5,5),(6,6); + +SELECT '*** BEFORE MUTATION BEFORE MERGE ***'; +SELECT id,a,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +set mutations_sync=1; +ALTER TABLE t_block_offset UPDATE a=0 WHERE id<4; + +SELECT '*** AFTER MUTATION BEFORE MERGE ***'; +SELECT id,a,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +OPTIMIZE TABLE t_block_offset FINAL; + +SELECT '*** AFTER MUTATION AFTER MERGE ***'; +SELECT *,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +INSERT INTO t_block_offset(id,a) VALUES (7,7),(8,8),(9,9); + +SELECT '*** AFTER MUTATION AFTER MERGE , NEW BLOCK ***'; +SELECT *,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +OPTIMIZE TABLE t_block_offset FINAL; + +SELECT '*** AFTER MUTATION AFTER MERGE , NEW BLOCK MERGED ***'; +SELECT *,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +DROP TABLE t_block_offset; + +{% endfor %} diff --git a/tests/queries/0_stateless/03001_block_offset_column_2.reference b/tests/queries/0_stateless/03001_block_offset_column_2.reference new file mode 100644 index 00000000000..a7ad8232678 --- /dev/null +++ b/tests/queries/0_stateless/03001_block_offset_column_2.reference @@ -0,0 +1,49 @@ +all_1_2_1 1 0 0 0 +all_1_2_1 1 1 1 2 +all_1_2_1 1 2 2 4 +all_1_2_1 1 3 3 6 +all_1_2_1 1 4 4 8 +all_1_2_1 1 5 5 10 +all_1_2_1 1 6 6 12 +all_1_2_1 1 7 7 14 +all_1_2_1 1 8 8 16 +all_1_2_1 1 9 9 18 +all_1_2_1 1 10 10 20 +all_1_2_1 1 11 11 22 +all_1_2_1 1 12 12 24 +all_1_2_1 1 13 13 26 +all_1_2_1 1 14 14 28 +all_1_2_1 1 15 15 30 +=========== +all_1_3_2 1 0 0 0 +all_1_3_2 1 1 2 2 +all_1_3_2 1 2 4 4 +all_1_3_2 1 3 6 6 +all_1_3_2 1 4 8 8 +all_1_3_2 1 5 10 10 +all_1_3_2 1 6 12 12 +all_1_3_2 1 7 14 14 +all_1_3_2 1 8 16 16 +all_1_3_2 1 9 18 18 +all_1_3_2 1 10 20 20 +all_1_3_2 1 11 22 22 +all_1_3_2 1 12 24 24 +all_1_3_2 1 13 26 26 +all_1_3_2 1 14 28 28 +all_1_3_2 1 15 30 30 +all_1_3_2 3 0 1 1 +all_1_3_2 3 1 3 3 +all_1_3_2 3 2 5 5 +all_1_3_2 3 3 7 7 +all_1_3_2 3 4 9 9 +all_1_3_2 3 5 11 11 +all_1_3_2 3 6 13 13 +all_1_3_2 3 7 15 15 +all_1_3_2 3 8 17 17 +all_1_3_2 3 9 19 19 +all_1_3_2 3 10 21 21 +all_1_3_2 3 11 23 23 +all_1_3_2 3 12 25 25 +all_1_3_2 3 13 27 27 +all_1_3_2 3 14 29 29 +all_1_3_2 3 15 31 31 diff --git a/tests/queries/0_stateless/03001_block_offset_column_2.sql b/tests/queries/0_stateless/03001_block_offset_column_2.sql new file mode 100644 index 00000000000..b994e37b952 --- /dev/null +++ b/tests/queries/0_stateless/03001_block_offset_column_2.sql @@ -0,0 +1,25 @@ + +DROP TABLE IF EXISTS t_block_offset; + +CREATE TABLE t_block_offset (id UInt32) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; + +INSERT INTO t_block_offset SELECT number * 2 FROM numbers(8); + +INSERT INTO t_block_offset SELECT number * 2 FROM numbers(8, 8); + +OPTIMIZE TABLE t_block_offset FINAL; + +SELECT _part, _block_number, _block_offset, _part_offset, id FROM t_block_offset ORDER BY _block_number, _block_offset; + +ALTER TABLE t_block_offset MODIFY SETTING enable_block_number_column = 1; +ALTER TABLE t_block_offset MODIFY SETTING enable_block_offset_column = 1; + +INSERT INTO t_block_offset SELECT number * 2 + 1 FROM numbers(16); + +OPTIMIZE TABLE t_block_offset FINAL; + +SELECT '==========='; +SELECT _part, _block_number, _block_offset, _part_offset, id FROM t_block_offset ORDER BY _block_number, _block_offset; + + +DROP TABLE t_block_offset; diff --git a/tests/queries/0_stateless/03001_data_version_column.sql b/tests/queries/0_stateless/03001_data_version_column.sql index 6cb6b192326..4e3377ebf47 100644 --- a/tests/queries/0_stateless/03001_data_version_column.sql +++ b/tests/queries/0_stateless/03001_data_version_column.sql @@ -5,16 +5,16 @@ CREATE TABLE t_data_version (a UInt64, b UInt64) ENGINE = MergeTree ORDER BY a; INSERT INTO t_data_version VALUES (1, 1); INSERT INTO t_data_version VALUES (2, 2); -SELECT _part, _data_version, * FROM t_data_version ORDER BY a; +SELECT _part, _part_data_version, * FROM t_data_version ORDER BY a; ALTER TABLE t_data_version UPDATE b = a * 100 WHERE 1 SETTINGS mutations_sync = 2; -SELECT _part, _data_version, * FROM t_data_version ORDER BY a; +SELECT _part, _part_data_version, * FROM t_data_version ORDER BY a; INSERT INTO t_data_version VALUES (3, 3); --- Check part pruning. -SELECT _part, _data_version, * FROM t_data_version WHERE _data_version = 4 ORDER BY a SETTINGS max_rows_to_read = 1; +-- Check parts pruning. +SELECT _part, _part_data_version, * FROM t_data_version WHERE _part_data_version = 4 ORDER BY a SETTINGS max_rows_to_read = 1; DROP TABLE t_data_version; diff --git a/tests/queries/0_stateless/03002_sample_factor_where.reference b/tests/queries/0_stateless/03002_sample_factor_where.reference new file mode 100644 index 00000000000..2f0d8589603 --- /dev/null +++ b/tests/queries/0_stateless/03002_sample_factor_where.reference @@ -0,0 +1,3 @@ +2 +0 +0 diff --git a/tests/queries/0_stateless/03002_sample_factor_where.sql b/tests/queries/0_stateless/03002_sample_factor_where.sql new file mode 100644 index 00000000000..6430034349a --- /dev/null +++ b/tests/queries/0_stateless/03002_sample_factor_where.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS t_sample_factor; + +CREATE TABLE t_sample_factor(a UInt64, b UInt64) ENGINE = MergeTree ORDER BY (a, b) SAMPLE BY b; +INSERT INTO t_sample_factor(a, b) VALUES (1, 2), (3, 4); + +SELECT uniq(b) * any(_sample_factor) FROM t_sample_factor SAMPLE 200000; + +SELECT uniq(b) * any(_sample_factor) FROM t_sample_factor SAMPLE 200000 WHERE a < -1; +SELECT uniq(b) * any(_sample_factor) FROM t_sample_factor SAMPLE 200000 PREWHERE a < -1; + +DROP TABLE t_sample_factor; From 0f0ea422f21af8e37aa5c8ef58002d608cde5c77 Mon Sep 17 00:00:00 2001 From: serxa Date: Thu, 7 Mar 2024 17:05:54 +0000 Subject: [PATCH 0084/1165] separate limits on number of waiting and executing queries --- .../settings.md | 16 ++- programs/server/Server.cpp | 1 + src/Common/AsyncLoader.cpp | 21 +++- src/Common/AsyncLoader.h | 65 ++++++++++- src/Common/tests/gtest_async_loader.cpp | 66 +++++++++++ src/Core/ServerSettings.h | 1 + src/Interpreters/ProcessList.cpp | 103 +++++++++++++++--- src/Interpreters/ProcessList.h | 41 ++++++- .../System/StorageSystemServerSettings.cpp | 1 + 9 files changed, 293 insertions(+), 22 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 07c9a2b88ab..63fbd9d1964 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -379,6 +379,18 @@ Type: UInt64 Default: 0 +## max_waiting_queries + +Limit on total number of concurrently waiting queries. Execution of a waiting query is blocked while required tables are loading asynchronously (see `async_load_databases`). Note that waiting queries are not counted when `max_concurrent_queries`, `max_concurrent_insert_queries`, `max_concurrent_select_queries`, `max_concurrent_queries_for_user` and `max_concurrent_queries_for_all_users` limits are checked. This correction is done to avoid hitting these limits just after server startup. Zero means unlimited. + +:::note +This setting can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged. +::: + +Type: UInt64 + +Default: 0 + ## max_connections Max server connections. @@ -1725,7 +1737,7 @@ Default value: `0.5`. Asynchronous loading of databases and tables. -If `true` all non-system databases with `Ordinary`, `Atomic` and `Replicated` engine will be loaded asynchronously after the ClickHouse server start up. See `system.asynchronous_loader` table, `tables_loader_background_pool_size` and `tables_loader_foreground_pool_size` server settings. Any query that tries to access a table, that is not yet loaded, will wait for exactly this table to be started up. If load job fails, query will rethrow an error (instead of shutting down the whole server in case of `async_load_databases = false`). The table that is waited for by at least one query will be loaded with higher priority. DDL queries on a database will wait for exactly that database to be started up. +If `true` all non-system databases with `Ordinary`, `Atomic` and `Replicated` engine will be loaded asynchronously after the ClickHouse server start up. See `system.asynchronous_loader` table, `tables_loader_background_pool_size` and `tables_loader_foreground_pool_size` server settings. Any query that tries to access a table, that is not yet loaded, will wait for exactly this table to be started up. If load job fails, query will rethrow an error (instead of shutting down the whole server in case of `async_load_databases = false`). The table that is waited for by at least one query will be loaded with higher priority. DDL queries on a database will wait for exactly that database to be started up. Also consider setting a limit `max_waiting_queries` for the total number of waiting queries. If `false`, all databases are loaded when the server starts. @@ -2926,7 +2938,7 @@ Default: 0 ## ignore_empty_sql_security_in_create_view_query {#ignore_empty_sql_security_in_create_view_query} -If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. +If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. :::note This setting is only necessary for the migration period and will become obsolete in 24.4 diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index a10f47be0b8..336563665a2 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1429,6 +1429,7 @@ try global_context->getProcessList().setMaxSize(new_server_settings.max_concurrent_queries); global_context->getProcessList().setMaxInsertQueriesAmount(new_server_settings.max_concurrent_insert_queries); global_context->getProcessList().setMaxSelectQueriesAmount(new_server_settings.max_concurrent_select_queries); + global_context->getProcessList().setMaxWaitingQueriesAmount(new_server_settings.max_waiting_queries); if (config->has("keeper_server")) global_context->updateKeeperConfiguration(*config); diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index 140194e10b4..80e4c72f1c1 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -140,6 +140,11 @@ void LoadJob::finish() finish_time = std::chrono::system_clock::now(); if (waiters > 0) finished.notify_all(); + else + { + on_waiters_increment = {}; + on_waiters_decrement = {}; + } } void LoadJob::scheduled(UInt64 job_id_) @@ -765,11 +770,25 @@ void AsyncLoader::wait(std::unique_lock & job_lock, const LoadJobPtr if (job->load_status != LoadStatus::PENDING) // Shortcut just to avoid incrementing ProfileEvents return; + if (job->on_waiters_increment) + job->on_waiters_increment(job); + + // WARNING: it is important not to throw below this point to avoid `on_waiters_increment` call w/o matching `on_waiters_decrement` call + Stopwatch watch; job->waiters++; job->finished.wait(job_lock, [&] { return job->load_status != LoadStatus::PENDING; }); job->waiters--; ProfileEvents::increment(ProfileEvents::AsyncLoaderWaitMicroseconds, watch.elapsedMicroseconds()); + + if (job->on_waiters_decrement) + job->on_waiters_decrement(job); + + if (job->waiters == 0) + { + job->on_waiters_increment = {}; + job->on_waiters_decrement = {}; + } } bool AsyncLoader::canSpawnWorker(Pool & pool, std::unique_lock &) @@ -859,7 +878,7 @@ void AsyncLoader::worker(Pool & pool) try { current_load_job = job.get(); - SCOPE_EXIT({ current_load_job = nullptr; }); // Note that recursive job execution is not supported + SCOPE_EXIT({ current_load_job = nullptr; }); // Note that recursive job execution is not supported, but jobs can wait one another job->execute(*this, pool_id, job); exception_from_job = {}; } diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h index b1b336d24dc..3f81a36aa96 100644 --- a/src/Common/AsyncLoader.h +++ b/src/Common/AsyncLoader.h @@ -59,7 +59,8 @@ enum class LoadStatus class LoadJob : private boost::noncopyable { public: - template + // NOTE: makeLoadJob() helper should be used instead of direct ctor call + template LoadJob(LoadJobSetType && dependencies_, String name_, size_t pool_id_, DFFunc && dependency_failure_, Func && func_) : dependencies(std::forward(dependencies_)) , name(std::move(name_)) @@ -69,6 +70,19 @@ public: , func(std::forward(func_)) {} + // NOTE: makeLoadJob() helper should be used instead of direct ctor call + template + LoadJob(LoadJobSetType && dependencies_, String name_, size_t pool_id_, WIFunc && on_waiters_increment_, WDFunc && on_waiters_decrement_, DFFunc && dependency_failure_, Func && func_) + : dependencies(std::forward(dependencies_)) + , name(std::move(name_)) + , execution_pool_id(pool_id_) + , pool_id(pool_id_) + , on_waiters_increment(std::forward(on_waiters_increment_)) + , on_waiters_decrement(std::forward(on_waiters_decrement_)) + , dependency_failure(std::forward(dependency_failure_)) + , func(std::forward(func_)) + {} + // Current job status. LoadStatus status() const; std::exception_ptr exception() const; @@ -112,6 +126,13 @@ private: std::atomic execution_pool_id; std::atomic pool_id; + // Handlers that is called by every new waiting thread, just before going to sleep. + // If `on_waiters_increment` throws, then wait is canceled, and corresponding `on_waiters_decrement` will never be called. + // It can be used for counting and limits on number of waiters. + // Note that implementations are called under `LoadJob::mutex` and should be fast. + std::function on_waiters_increment; + std::function on_waiters_decrement; + // Handler for failed or canceled dependencies. // If job needs to be canceled on `dependency` failure, then function should set `cancel` to a specific reason. // Note that implementation should be fast and cannot use AsyncLoader, because it is called under `AsyncLoader::mutex`. @@ -140,8 +161,50 @@ void cancelOnDependencyFailure(const LoadJobPtr & self, const LoadJobPtr & depen void ignoreDependencyFailure(const LoadJobPtr & self, const LoadJobPtr & dependency, std::exception_ptr & cancel); template concept LoadJobDependencyFailure = std::invocable; +template concept LoadJobOnWaiters = std::invocable; template concept LoadJobFunc = std::invocable; +LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) +{ + return std::make_shared(std::move(dependencies), std::move(name), 0, on_waiters_increment, on_waiters_decrement, std::forward(dependency_failure), std::forward(func)); +} + +LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) +{ + return std::make_shared(dependencies, std::move(name), 0, on_waiters_increment, on_waiters_decrement, std::forward(dependency_failure), std::forward(func)); +} + +LoadJobPtr makeLoadJob(LoadJobSet && dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) +{ + return std::make_shared(std::move(dependencies), std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, std::forward(dependency_failure), std::forward(func)); +} + +LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) +{ + return std::make_shared(dependencies, std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, std::forward(dependency_failure), std::forward(func)); +} + +LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func) +{ + return std::make_shared(std::move(dependencies), std::move(name), 0, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward(func)); +} + +LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func) +{ + return std::make_shared(dependencies, std::move(name), 0, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward(func)); +} + +LoadJobPtr makeLoadJob(LoadJobSet && dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func) +{ + return std::make_shared(std::move(dependencies), std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward(func)); +} + +LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func) +{ + return std::make_shared(dependencies, std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward(func)); +} + + LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) { return std::make_shared(std::move(dependencies), std::move(name), 0, std::forward(dependency_failure), std::forward(func)); diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp index fc2537abcfc..62a27f259cc 100644 --- a/src/Common/tests/gtest_async_loader.cpp +++ b/src/Common/tests/gtest_async_loader.cpp @@ -643,6 +643,72 @@ TEST(AsyncLoader, CustomDependencyFailure) ASSERT_EQ(good_count.load(), 3); } +TEST(AsyncLoader, WaitersLimit) +{ + AsyncLoaderTest t(16); + + std::atomic waiters_total{0}; + int waiters_limit = 5; + auto waiters_inc = [&] (const LoadJobPtr &) { + int value = waiters_total.load(); + while (true) + { + if (value >= waiters_limit) + throw Exception(ErrorCodes::ASYNC_LOAD_FAILED, "Too many waiters: {}", value); + if (waiters_total.compare_exchange_strong(value, value + 1)) + break; + } + }; + auto waiters_dec = [&] (const LoadJobPtr &) { + waiters_total.fetch_sub(1); + }; + + std::barrier sync(2); + t.loader.start(); + + auto job_func = [&] (AsyncLoader &, const LoadJobPtr &) { + sync.arrive_and_wait(); // (A) + }; + + auto job = makeLoadJob({}, "job", waiters_inc, waiters_dec, job_func); + auto task = t.schedule({job}); + + std::atomic failure{0}; + std::atomic success{0}; + std::vector waiters; + waiters.reserve(10); + auto waiter = [&] { + try + { + t.loader.wait(job); + success.fetch_add(1); + } + catch(...) + { + failure.fetch_add(1); + } + }; + + for (int i = 0; i < 10; i++) + waiters.emplace_back(waiter); + + while (failure.load() != 5) + std::this_thread::yield(); + + ASSERT_EQ(job->waitersCount(), 5); + + sync.arrive_and_wait(); // (A) + + for (auto & thread : waiters) + thread.join(); + + ASSERT_EQ(success.load(), 5); + ASSERT_EQ(failure.load(), 5); + ASSERT_EQ(waiters_total.load(), 0); + + t.loader.wait(); +} + TEST(AsyncLoader, TestConcurrency) { AsyncLoaderTest t(10); diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index c82255ec59c..129b1016fca 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -64,6 +64,7 @@ namespace DB M(UInt64, max_concurrent_queries, 0, "Maximum number of concurrently executed queries. Zero means unlimited.", 0) \ M(UInt64, max_concurrent_insert_queries, 0, "Maximum number of concurrently INSERT queries. Zero means unlimited.", 0) \ M(UInt64, max_concurrent_select_queries, 0, "Maximum number of concurrently SELECT queries. Zero means unlimited.", 0) \ + M(UInt64, max_waiting_queries, 0, "Maximum number of concurrently waiting queries blocked due to `async_load_databases`. Note that waiting queries are not considered by `max_concurrent_*queries*` limits. Zero means unlimited.", 0) \ \ M(Double, cache_size_to_ram_max_ratio, 0.5, "Set cache size ro RAM max ratio. Allows to lower cache size on low-memory systems.", 0) \ M(String, uncompressed_cache_policy, DEFAULT_UNCOMPRESSED_CACHE_POLICY, "Uncompressed cache policy name.", 0) \ diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 3bd7b2d4206..f451d561e60 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -83,25 +83,31 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q IAST::QueryKind query_kind = ast->getQueryKind(); const auto queue_max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds(); - if (!is_unlimited_query && max_size && processes.size() >= max_size) + UInt64 waiting_queries = waiting_queries_amount.load(); + if (!is_unlimited_query && max_size && processes.size() >= max_size + waiting_queries) { if (queue_max_wait_ms) LOG_WARNING(getLogger("ProcessList"), "Too many simultaneous queries, will wait {} ms.", queue_max_wait_ms); - if (!queue_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(queue_max_wait_ms), [&]{ return processes.size() < max_size; })) - throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, "Too many simultaneous queries. Maximum: {}", max_size); + if (!queue_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(queue_max_wait_ms), + [&]{ waiting_queries = waiting_queries_amount.load(); return processes.size() < max_size + waiting_queries; })) + throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, + "Too many simultaneous queries. Maximum: {}{}", + max_size, waiting_queries == 0 ? "" : fmt::format(", waiting: {}", waiting_queries)); } if (!is_unlimited_query) { QueryAmount amount = getQueryKindAmount(query_kind); - if (max_insert_queries_amount && query_kind == IAST::QueryKind::Insert && amount >= max_insert_queries_amount) + UInt64 waiting_inserts = waiting_insert_queries_amount.load(); + UInt64 waiting_selects = waiting_select_queries_amount.load(); + if (max_insert_queries_amount && query_kind == IAST::QueryKind::Insert && amount >= max_insert_queries_amount + waiting_inserts) throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, - "Too many simultaneous insert queries. Maximum: {}, current: {}", - max_insert_queries_amount, amount); - if (max_select_queries_amount && query_kind == IAST::QueryKind::Select && amount >= max_select_queries_amount) + "Too many simultaneous insert queries. Maximum: {}, current: {}{}", + max_insert_queries_amount, amount, waiting_inserts == 0 ? "" : fmt::format(", waiting: {}", waiting_inserts)); + if (max_select_queries_amount && query_kind == IAST::QueryKind::Select && amount >= max_select_queries_amount + waiting_selects) throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, - "Too many simultaneous select queries. Maximum: {}, current: {}", - max_select_queries_amount, amount); + "Too many simultaneous select queries. Maximum: {}, current: {}{}", + max_select_queries_amount, amount, waiting_selects == 0 ? "" : fmt::format(", waiting: {}", waiting_selects)); } { @@ -124,10 +130,12 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q * once is already processing 50+ concurrent queries (including analysts or any other users). */ + waiting_queries = waiting_queries_amount.load(); if (!is_unlimited_query && settings.max_concurrent_queries_for_all_users - && processes.size() >= settings.max_concurrent_queries_for_all_users) + && processes.size() >= settings.max_concurrent_queries_for_all_users + waiting_queries_amount) throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, "Too many simultaneous queries for all users. " - "Current: {}, maximum: {}", processes.size(), settings.max_concurrent_queries_for_all_users.toString()); + "Current: {}, maximum: {}{}", processes.size(), settings.max_concurrent_queries_for_all_users.toString(), + waiting_queries == 0 ? "" : fmt::format(", waiting: {}", waiting_queries)); } /** Why we use current user? @@ -145,13 +153,15 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q if (user_process_list != user_to_queries.end()) { + UInt64 user_waiting_queries = user_process_list->second.waiting_queries_amount.load(); if (!is_unlimited_query && settings.max_concurrent_queries_for_user - && user_process_list->second.queries.size() >= settings.max_concurrent_queries_for_user) + && user_process_list->second.queries.size() >= settings.max_concurrent_queries_for_user + user_waiting_queries) throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, "Too many simultaneous queries for user {}. " - "Current: {}, maximum: {}", + "Current: {}, maximum: {}{}", client_info.current_user, user_process_list->second.queries.size(), - settings.max_concurrent_queries_for_user.toString()); + settings.max_concurrent_queries_for_user.toString(), + user_waiting_queries == 0 ? "" : fmt::format(", waiting: {}", user_waiting_queries)); auto running_query = user_process_list->second.queries.find(client_info.current_query_id); @@ -745,4 +755,69 @@ ProcessList::QueryAmount ProcessList::getQueryKindAmount(const IAST::QueryKind & return found->second; } +void ProcessList::increaseWaitingQueryAmount(const QueryStatusPtr & status) +{ + UInt64 limit = max_waiting_queries_amount.load(); + UInt64 value = waiting_queries_amount.load(); + while (true) + { + if (value >= limit) + throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, + "Too many simultaneous waiting queries. Maximum: {}, waiting: {}", + limit, value); + if (waiting_queries_amount.compare_exchange_strong(value, value + 1)) + break; + } + + // WARNING: After this point we should not throw, otherwise corresponding `decreaseWaitingQueryAmount` will not be called. + + // Update query kind counters + if (status->query_kind == IAST::QueryKind::Insert) + waiting_insert_queries_amount.fetch_add(1); + if (status->query_kind == IAST::QueryKind::Select) + waiting_select_queries_amount.fetch_add(1); + + // Update per-user counter + status->getUserProcessList()->waiting_queries_amount.fetch_add(1); + + // We have to notify because some queries might be waiting on `have_space` + // and this query leaves its space by transitioning to waiting state + have_space.notify_all(); +} + +void ProcessList::decreaseWaitingQueryAmount(const QueryStatusPtr & status) +{ + if (status->getUserProcessList()->waiting_queries_amount.fetch_sub(1) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong insert waiting query amount for user: decrease to negative"); + + if (status->query_kind == IAST::QueryKind::Insert && waiting_insert_queries_amount.fetch_sub(1) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong insert waiting query amount: decrease to negative"); + + if (status->query_kind == IAST::QueryKind::Select && waiting_select_queries_amount.fetch_sub(1) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong select waiting query amount: decrease to negative"); + + if (waiting_queries_amount.fetch_sub(1) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong waiting query amount: decrease to negative"); +} + +void ProcessList::incrementWaiters() +{ + ContextPtr context = CurrentThread::getQueryContext(); + QueryStatusPtr status = context->getProcessListElement(); + + // Query became "waiting" with the first thread that waits + if (status->waiting_threads.fetch_add(1) == 0) + increaseWaitingQueryAmount(status); +} + +void ProcessList::decrementWaiters() +{ + ContextPtr context = CurrentThread::getQueryContext(); + QueryStatusPtr status = context->getProcessListElement(); + + // Query became "non-waiting" with the last thread that no longer waits + if (status->waiting_threads.fetch_sub(1) == 1) + decreaseWaitingQueryAmount(status); +} + } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 1c253f562e8..75350627698 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -42,10 +42,6 @@ class ThreadStatus; class ProcessListEntry; -/** List of currently executing queries. - * Also implements limit on their number. - */ - /** Information of process list element. * To output in SHOW PROCESSLIST query. Does not contain any complex objects, that do something on copy or destructor. */ @@ -114,8 +110,13 @@ protected: /// Including EndOfStream or Exception. std::atomic is_all_data_sent { false }; + /// Number of threads for the query that are waiting for load jobs + std::atomic waiting_threads{0}; + + /// For initialization of ProcessListForUser during process insertion. void setUserProcessList(ProcessListForUser * user_process_list_); /// Be careful using it. For example, queries field of ProcessListForUser could be modified concurrently. + ProcessListForUser * getUserProcessList() { return user_process_list; } const ProcessListForUser * getUserProcessList() const { return user_process_list; } /// Sets an entry in the ProcessList associated with this QueryStatus. @@ -283,6 +284,9 @@ struct ProcessListForUser /// Count network usage for all simultaneously running queries of single user. ThrottlerPtr user_throttler; + /// Number of queries waiting on load jobs + std::atomic waiting_queries_amount{0}; + ProcessListForUserInfo getInfo(bool get_profile_events = false) const; /// Clears MemoryTracker for the user. @@ -341,6 +345,9 @@ protected: }; +/** List of currently executing queries. + * Also implements limit on their number. + */ class ProcessList : public ProcessListBase { public: @@ -399,10 +406,21 @@ protected: /// amount of queries by query kind. QueryKindAmounts query_kind_amounts; + /// limit for waiting queries. 0 means no limit. Otherwise, when limit exceeded, an exception is thrown. + std::atomic max_waiting_queries_amount{0}; + + /// amounts of waiting queries + std::atomic waiting_queries_amount{0}; + std::atomic waiting_insert_queries_amount{0}; + std::atomic waiting_select_queries_amount{0}; + void increaseQueryKindAmount(const IAST::QueryKind & query_kind); void decreaseQueryKindAmount(const IAST::QueryKind & query_kind); QueryAmount getQueryKindAmount(const IAST::QueryKind & query_kind) const; + void increaseWaitingQueryAmount(const QueryStatusPtr & status); + void decreaseWaitingQueryAmount(const QueryStatusPtr & status); + public: using EntryPtr = std::shared_ptr; @@ -458,6 +476,21 @@ public: return max_select_queries_amount; } + void setMaxWaitingQueriesAmount(UInt64 max_waiting_queries_amount_) + { + max_waiting_queries_amount.store(max_waiting_queries_amount_); + // NOTE: We cannot cancel waiting queries when limit is lowered. They have to wait anyways, but new queries will be canceled instead of waiting. + } + + size_t getMaxWaitingQueriesAmount() const + { + return max_waiting_queries_amount.load(); + } + + // Handlers for AsyncLoader waiters + void incrementWaiters(); + void decrementWaiters(); + /// Try call cancel() for input and output streams of query with specified id and user CancellationCode sendCancelToQuery(const String & current_query_id, const String & current_user, bool kill = false); CancellationCode sendCancelToQuery(QueryStatusPtr elem, bool kill = false); diff --git a/src/Storages/System/StorageSystemServerSettings.cpp b/src/Storages/System/StorageSystemServerSettings.cpp index f390985546b..bf14f757a19 100644 --- a/src/Storages/System/StorageSystemServerSettings.cpp +++ b/src/Storages/System/StorageSystemServerSettings.cpp @@ -70,6 +70,7 @@ void StorageSystemServerSettings::fillData(MutableColumns & res_columns, Context {"max_concurrent_queries", {std::to_string(context->getProcessList().getMaxSize()), ChangeableWithoutRestart::Yes}}, {"max_concurrent_insert_queries", {std::to_string(context->getProcessList().getMaxInsertQueriesAmount()), ChangeableWithoutRestart::Yes}}, {"max_concurrent_select_queries", {std::to_string(context->getProcessList().getMaxSelectQueriesAmount()), ChangeableWithoutRestart::Yes}}, + {"max_waiting_queries", {std::to_string(context->getProcessList().getMaxWaitingQueriesAmount()), ChangeableWithoutRestart::Yes}}, {"background_buffer_flush_schedule_pool_size", {std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundBufferFlushSchedulePoolSize)), ChangeableWithoutRestart::IncreaseOnly}}, {"background_schedule_pool_size", {std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundSchedulePoolSize)), ChangeableWithoutRestart::IncreaseOnly}}, From b0b38121a60682ee79d47914e18dd57d892f394e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 7 Mar 2024 20:44:03 +0000 Subject: [PATCH 0085/1165] Crash repro --- src/Storages/StorageMergeTree.cpp | 1 + ...eplicas_cte_explain_syntax_crash.reference | 15 +++++++ ...llel_replicas_cte_explain_syntax_crash.sql | 44 +++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.reference create mode 100644 tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index e15b308f084..dbc901d4a5f 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -220,6 +220,7 @@ void StorageMergeTree::read( Block header; if (local_context->getSettingsRef().allow_experimental_analyzer) { + chassert(query_info.query_tree); QueryTreeNodePtr modified_query_tree = query_info.query_tree->clone(); rewriteJoinToGlobalJoin(modified_query_tree, local_context); modified_query_tree = buildQueryTreeForShard(query_info.planner_context, modified_query_tree); diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.reference b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.reference new file mode 100644 index 00000000000..078890722fd --- /dev/null +++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.reference @@ -0,0 +1,15 @@ +WITH + cte1 AS + ( + SELECT n + FROM numbers_1e6__fuzz_34 + ), + cte2 AS + ( + SELECT n + FROM numbers_1e6__fuzz_33 + PREWHERE n IN cte1 + ) +SELECT count() +FROM +cte2 diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql new file mode 100644 index 00000000000..a407fceb1c6 --- /dev/null +++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql @@ -0,0 +1,44 @@ +DROP TABLE IF EXISTS numbers_1e6__fuzz_34; +DROP TABLE IF EXISTS numbers_1e6__fuzz_33; + +CREATE TABLE numbers_1e6__fuzz_34 +( + `n` LowCardinality(Nullable(UInt8)) +) +ENGINE = MergeTree +ORDER BY n +SETTINGS allow_nullable_key = 1 +AS SELECT * +FROM numbers(1000000) +SETTINGS allow_suspicious_low_cardinality_types = 1; + + +CREATE TABLE numbers_1e6__fuzz_33 +( + `n` LowCardinality(Nullable(UInt8)) +) +ENGINE = MergeTree +ORDER BY n +SETTINGS allow_nullable_key = 1 +AS SELECT * +FROM numbers(1000000) +SETTINGS allow_suspicious_low_cardinality_types = 1; + +SET allow_experimental_analyzer = 0; +SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +EXPLAIN SYNTAX +WITH + cte1 AS + ( + SELECT n + FROM numbers_1e6__fuzz_34 + ), + cte2 AS + ( + SELECT n + FROM numbers_1e6__fuzz_33 + PREWHERE n IN (cte1) + ) +SELECT count() +FROM cte2; +-- SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; From cde811804f68656577dd1497e511a4a9295d4544 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 8 Mar 2024 12:20:34 +0800 Subject: [PATCH 0086/1165] prevent memcpySmallAllowReadWriteOverflow15Impl optimized to memcpy --- src/Common/memcpySmall.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Common/memcpySmall.h b/src/Common/memcpySmall.h index f3d26c60380..f5e9f31fc46 100644 --- a/src/Common/memcpySmall.h +++ b/src/Common/memcpySmall.h @@ -49,6 +49,9 @@ namespace detail dst += 16; src += 16; n -= 16; + + /// Avoid clang loop-idion optimization, which transforms _mm_storeu_si128 to built-in memcpy + __asm__ __volatile__("" : : : "memory"); } } } From 89ae39e598a481dbb2c610ee7dca1fc7272517b7 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 8 Mar 2024 12:22:07 +0800 Subject: [PATCH 0087/1165] optimize column string replicate --- src/Columns/ColumnString.cpp | 38 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index b9128372cea..0c52a7be086 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -460,6 +460,7 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, Per DefaultPartialSort()); } + ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const { size_t col_size = size(); @@ -471,32 +472,35 @@ ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const if (0 == col_size) return res; - Chars & res_chars = res->chars; Offsets & res_offsets = res->offsets; - res_chars.reserve_exact(chars.size() / col_size * replicate_offsets.back()); - res_offsets.reserve_exact(replicate_offsets.back()); - - Offset prev_replicate_offset = 0; - Offset prev_string_offset = 0; - Offset current_new_offset = 0; + res_offsets.resize_exact(replicate_offsets.back()); + Chars & res_chars = res->chars; + size_t res_chars_size = 0; for (size_t i = 0; i < col_size; ++i) { - size_t size_to_replicate = replicate_offsets[i] - prev_replicate_offset; - size_t string_size = offsets[i] - prev_string_offset; + size_t size_to_replicate = replicate_offsets[i] - replicate_offsets[i - 1]; + size_t string_size = offsets[i] - offsets[i - 1]; + res_chars_size += size_to_replicate * string_size; + } + res_chars.resize_exact(res_chars_size); + size_t curr_row = 0; + size_t curr_offset = 0; + for (size_t i = 0; i < col_size; ++i) + { + const size_t size_to_replicate = replicate_offsets[i] - replicate_offsets[i - 1]; + const size_t string_size = offsets[i] - offsets[i-1]; + const UInt8 * src = &chars[offsets[i - 1]]; for (size_t j = 0; j < size_to_replicate; ++j) { - current_new_offset += string_size; - res_offsets.push_back(current_new_offset); - - res_chars.resize(res_chars.size() + string_size); memcpySmallAllowReadWriteOverflow15( - &res_chars[res_chars.size() - string_size], &chars[prev_string_offset], string_size); - } + &res_chars[curr_offset], src, string_size); - prev_replicate_offset = replicate_offsets[i]; - prev_string_offset = offsets[i]; + curr_offset += string_size; + res_offsets[curr_row] = curr_offset; + ++curr_row; + } } return res; From 391af00b64e62dc7d9e1c5726c1aa202f19dffd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=89=AC?= <654010905@qq.com> Date: Fri, 8 Mar 2024 13:47:35 +0800 Subject: [PATCH 0088/1165] Update memcpySmall.h --- src/Common/memcpySmall.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/memcpySmall.h b/src/Common/memcpySmall.h index f5e9f31fc46..90648254d76 100644 --- a/src/Common/memcpySmall.h +++ b/src/Common/memcpySmall.h @@ -50,7 +50,7 @@ namespace detail src += 16; n -= 16; - /// Avoid clang loop-idion optimization, which transforms _mm_storeu_si128 to built-in memcpy + /// Avoid clang loop-idiom optimization, which transforms _mm_storeu_si128 to built-in memcpy __asm__ __volatile__("" : : : "memory"); } } From e2317477f7b95d07407db8def968d286aa9e270d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 8 Mar 2024 17:12:31 +0100 Subject: [PATCH 0089/1165] fix removing is_active node after re-creation --- src/Databases/DatabaseReplicatedWorker.cpp | 2 ++ tests/integration/test_replicated_database/test.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 2056b403ff6..0a6e8f9345e 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -75,6 +75,8 @@ void DatabaseReplicatedDDLWorker::initializeReplication() String active_path = fs::path(database->replica_path) / "active"; String active_id = toString(ServerUUID::get()); zookeeper->deleteEphemeralNodeIfContentMatches(active_path, active_id); + if (active_node_holder) + active_node_holder->setAlreadyRemoved(); zookeeper->create(active_path, active_id, zkutil::CreateMode::Ephemeral); active_node_holder.reset(); active_node_holder_zookeeper = zookeeper; diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index b47f86a843d..4f449f9a296 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1141,6 +1141,8 @@ def test_sync_replica(started_cluster): dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_sync_database") + assert "2\n" == main_node.query("SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'") + assert dummy_node.query( "SELECT count() FROM system.tables where database='test_sync_database'" ).strip() == str(number_of_tables) From babe00003620ca34f228009d919d5613db867dee Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 8 Mar 2024 16:43:10 +0000 Subject: [PATCH 0090/1165] Automatic style fix --- tests/integration/test_replicated_database/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 4f449f9a296..881659262ac 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1141,7 +1141,9 @@ def test_sync_replica(started_cluster): dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_sync_database") - assert "2\n" == main_node.query("SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'") + assert "2\n" == main_node.query( + "SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'" + ) assert dummy_node.query( "SELECT count() FROM system.tables where database='test_sync_database'" From bd530a175301860ba1ed13b5b97c56f12e7b13ce Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 8 Mar 2024 20:21:28 +0000 Subject: [PATCH 0091/1165] Forgot to enable analyzer --- .../03006_parallel_replicas_cte_explain_syntax_crash.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql index a407fceb1c6..f46817d5e82 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql +++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql @@ -24,7 +24,7 @@ AS SELECT * FROM numbers(1000000) SETTINGS allow_suspicious_low_cardinality_types = 1; -SET allow_experimental_analyzer = 0; +SET allow_experimental_analyzer = 1; SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; EXPLAIN SYNTAX WITH @@ -41,4 +41,3 @@ WITH ) SELECT count() FROM cte2; --- SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; From 2ce96f48f3c3958ef51c3e620b886d633436bb26 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 8 Mar 2024 21:58:01 +0100 Subject: [PATCH 0092/1165] Update 02962_system_sync_replica_lightweight_from_modifier.sh --- .../02962_system_sync_replica_lightweight_from_modifier.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02962_system_sync_replica_lightweight_from_modifier.sh b/tests/queries/0_stateless/02962_system_sync_replica_lightweight_from_modifier.sh index f47801abf73..b61be87411d 100755 --- a/tests/queries/0_stateless/02962_system_sync_replica_lightweight_from_modifier.sh +++ b/tests/queries/0_stateless/02962_system_sync_replica_lightweight_from_modifier.sh @@ -14,7 +14,7 @@ export REPLICAS_TO_DROP for i in $(seq $TOTAL_REPLICAS); do $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_table_$i" - $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table_$i (key UInt64, value UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table', '$i') ORDER BY key" + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table_$i (key UInt64, value UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table', '$i') ORDER BY key SETTINGS old_parts_lifetime=1" done function insert_thread() { @@ -35,7 +35,7 @@ function sync_and_drop_replicas() { done for i in $(seq $REPLICAS_TO_DROP); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table_$i (key UInt64, value UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table', '$i') ORDER BY key" + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table_$i (key UInt64, value UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table', '$i') ORDER BY key SETTINGS old_parts_lifetime=1" done done } @@ -87,4 +87,4 @@ for i in $(seq $TOTAL_REPLICAS); do if [ $i -gt $REPLICAS_TO_DROP ]; then $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_table_$i" fi -done \ No newline at end of file +done From b07e5d9f5a6420a9ba3d19d63070831a9db121ea Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 10 Mar 2024 08:11:59 +0300 Subject: [PATCH 0093/1165] Update StorageMergeTree.cpp --- src/Storages/StorageMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 0748ac2dbdf..055a48ad998 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2083,7 +2083,7 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con MergeTreePartInfo dst_part_info(partition_id, temp_index, temp_index, src_part->info.level); IDataPartStorage::ClonePartParams clone_params{.txn = local_context->getCurrentTransaction()}; - LOG_TRACE(log, "Partition exps are the same:part id: {}; number of disks:{}",dst_part_info.partition_id, this->getStoragePolicy()->getDisks().size()); + bool on_same_disk = false; for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) if (disk->getName() == src_part->getDataPartStorage().getDiskName()) From bc936cc7bba34f63d42c3b5c62c0936330bac3a7 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 11 Mar 2024 14:41:51 +0800 Subject: [PATCH 0094/1165] inline small function in PODArrayDetails --- src/Common/PODArray.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/PODArray.cpp b/src/Common/PODArray.cpp index dd1fed08cb5..2786a50346a 100644 --- a/src/Common/PODArray.cpp +++ b/src/Common/PODArray.cpp @@ -22,7 +22,7 @@ void protectMemoryRegion(void * addr, size_t len, int prot) } #endif -size_t byte_size(size_t num_elements, size_t element_size) +ALWAY_INLINE size_t byte_size(size_t num_elements, size_t element_size) { size_t amount; if (__builtin_mul_overflow(num_elements, element_size, &amount)) From 150e7f338a3a1d49104f4d605bd2f65bd20fa149 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 11 Mar 2024 14:44:16 +0800 Subject: [PATCH 0095/1165] inline another function --- src/Common/PODArray.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/PODArray.cpp b/src/Common/PODArray.cpp index 2786a50346a..a8195f15241 100644 --- a/src/Common/PODArray.cpp +++ b/src/Common/PODArray.cpp @@ -22,7 +22,7 @@ void protectMemoryRegion(void * addr, size_t len, int prot) } #endif -ALWAY_INLINE size_t byte_size(size_t num_elements, size_t element_size) +ALWAYS_INLINE size_t byte_size(size_t num_elements, size_t element_size) { size_t amount; if (__builtin_mul_overflow(num_elements, element_size, &amount)) @@ -30,7 +30,7 @@ ALWAY_INLINE size_t byte_size(size_t num_elements, size_t element_size) return amount; } -size_t minimum_memory_for_elements(size_t num_elements, size_t element_size, size_t pad_left, size_t pad_right) +ALWAYS_INLINE size_t minimum_memory_for_elements(size_t num_elements, size_t element_size, size_t pad_left, size_t pad_right) { size_t amount; if (__builtin_add_overflow(byte_size(num_elements, element_size), pad_left + pad_right, &amount)) From 8a11afeba1e80f3f0210af5d19338550f631ceab Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 11 Mar 2024 11:44:34 +0100 Subject: [PATCH 0096/1165] Updated settings changed history --- src/Core/SettingsChangesHistory.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index b68789d5f43..d458f935edf 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -93,6 +93,12 @@ static std::map sett {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"}, {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."}, {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, + {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."}, + {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."}, + {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."}, + {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."}, + {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."}, + {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, @@ -122,7 +128,6 @@ static std::map sett {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."}, }}, {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, From e562d97ff59739da3ca3650a33644bf9700becd7 Mon Sep 17 00:00:00 2001 From: Nikolay Yankin <211292+kolya7k@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:07:19 +0300 Subject: [PATCH 0097/1165] Update install.md https://packages.clickhouse.com/tgz/stable/ is multi-paged now and sorted by date asc so we can't get very last version --- docs/en/getting-started/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 234420de374..3b01434ecc5 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -262,7 +262,7 @@ The required version can be downloaded with `curl` or `wget` from repository htt After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest stable version: ``` bash -LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ +LATEST_VERSION=$(curl -s -L https://api.github.com/repos/ClickHouse/ClickHouse/tags | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION From 2bec92c48e9ad01ecc1bde38808ac35e628758b0 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 11 Mar 2024 14:41:32 +0000 Subject: [PATCH 0098/1165] better _part_offset column --- .../MergeTree/MergeTreeRangeReader.cpp | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index eb213fdc5ad..bbc4adaf458 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -872,7 +872,7 @@ size_t MergeTreeRangeReader::currentMark() const return stream.currentMark(); } -const NameSet MergeTreeRangeReader::virtuals_to_fill = {"_part_offset", BlockOffsetColumn::name}; +const NameSet MergeTreeRangeReader::virtuals_to_fill = {"_part_offset", "_block_offset"}; size_t MergeTreeRangeReader::Stream::numPendingRows() const { @@ -1146,30 +1146,29 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t if (!result.rows_per_granule.empty()) result.adjustLastGranule(); - /// Column _block_offset is the same as _part_offset if it's not persisted in part. - bool has_part_offset = read_sample_block.has("_part_offset"); - bool has_block_offset = read_sample_block.has(BlockOffsetColumn::name); + ColumnPtr part_offset_column; - if (has_part_offset || has_block_offset) + auto add_offset_column = [&](const auto & column_name) { - auto part_offset_column = createPartOffsetColumn(result, leading_begin_part_offset, leading_end_part_offset); + size_t pos = read_sample_block.getPositionByName(column_name); + chassert(pos < result.columns.size()); - auto add_offset_column = [&](const auto & column_name) - { - size_t pos = read_sample_block.getPositionByName(column_name); - chassert(pos < result.columns.size()); + /// Column may be persisted in part. + if (result.columns[pos]) + return; - /// Column may be persisted in part. - if (!result.columns[pos]) - result.columns[pos] = part_offset_column; - }; + if (!part_offset_column) + part_offset_column = createPartOffsetColumn(result, leading_begin_part_offset, leading_end_part_offset); - if (has_part_offset) - add_offset_column("_part_offset"); + result.columns[pos] = part_offset_column; + }; - if (has_block_offset) - add_offset_column(BlockOffsetColumn::name); - } + if (read_sample_block.has("_part_offset")) + add_offset_column("_part_offset"); + + /// Column _block_offset is the same as _part_offset if it's not persisted in part. + if (read_sample_block.has(BlockOffsetColumn::name)) + add_offset_column(BlockOffsetColumn::name); return result; } From 96778ff6aec905172cb570599884b87b27ff832c Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 11 Mar 2024 16:09:14 +0100 Subject: [PATCH 0099/1165] Less contention in cache (Part 3) --- src/Common/ProfileEvents.cpp | 3 +- src/Interpreters/Cache/EvictionCandidates.cpp | 23 ++++++++++--- src/Interpreters/Cache/EvictionCandidates.h | 9 ++++- src/Interpreters/Cache/FileCache.cpp | 34 ++++++++++++------- src/Interpreters/Cache/IFileCachePriority.h | 9 +++-- .../Cache/LRUFileCachePriority.cpp | 26 +++++++++++--- src/Interpreters/Cache/LRUFileCachePriority.h | 5 +-- src/Interpreters/Cache/Metadata.cpp | 2 +- .../Cache/SLRUFileCachePriority.cpp | 31 +++++++++-------- .../Cache/SLRUFileCachePriority.h | 5 +-- 10 files changed, 102 insertions(+), 45 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index c1ac3d08245..7ca11a16cc2 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -459,7 +459,8 @@ The server successfully detected this situation and will download merged part fr M(FilesystemCacheLoadMetadataMicroseconds, "Time spent loading filesystem cache metadata") \ M(FilesystemCacheEvictedBytes, "Number of bytes evicted from filesystem cache") \ M(FilesystemCacheEvictedFileSegments, "Number of file segments evicted from filesystem cache") \ - M(FilesystemCacheEvictionSkippedFileSegments, "Number of file segments skipped for eviction because of being unreleasable") \ + M(FilesystemCacheEvictionSkippedFileSegments, "Number of file segments skipped for eviction because of being in unreleasable state") \ + M(FilesystemCacheEvictionSkippedEvictingFileSegments, "Number of file segments skipped for eviction because of being in evicting state") \ M(FilesystemCacheEvictionTries, "Number of filesystem cache eviction attempts") \ M(FilesystemCacheLockKeyMicroseconds, "Lock cache key time") \ M(FilesystemCacheLockMetadataMicroseconds, "Lock filesystem cache metadata time") \ diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 7dceab4f95f..fb0b3809f80 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -32,7 +32,7 @@ void EvictionCandidates::add(LockedKey & locked_key, const FileSegmentMetadataPt ++candidates_size; } -void EvictionCandidates::evict(FileCacheQueryLimit::QueryContext * query_context, const CacheGuard::Lock & lock) +void EvictionCandidates::evict() { if (candidates.empty()) return; @@ -59,14 +59,27 @@ void EvictionCandidates::evict(FileCacheQueryLimit::QueryContext * query_context ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedBytes, segment->range().size()); locked_key->removeFileSegment(segment->offset(), segment->lock()); - queue_it->remove(lock); - - if (query_context) - query_context->remove(segment->key(), segment->offset(), lock); + queue_it->invalidate(); to_evict.pop_back(); } } } +void EvictionCandidates::finalize(FileCacheQueryLimit::QueryContext * query_context, const CacheGuard::Lock & lock) +{ + for (const auto & it : invalidated_queue_entries) + { + if (query_context) + { + const auto & entry = it->getEntry(); + query_context->remove(entry->key, entry->offset, lock); + } + it->remove(lock); + } + + if (finalize_eviction_func) + finalize_eviction_func(lock); +} + } diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index 0557962d97f..7859762be09 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -11,7 +11,9 @@ public: void add(LockedKey & locked_key, const FileSegmentMetadataPtr & candidate); - void evict(FileCacheQueryLimit::QueryContext * query_context, const CacheGuard::Lock &); + void evict(); + + void finalize(FileCacheQueryLimit::QueryContext * query_context, const CacheGuard::Lock & lock); size_t size() const { return candidates_size; } @@ -19,6 +21,9 @@ public: auto end() const { return candidates.end(); } + using FinalizeEvictionFunc = std::function; + void setFinalizeEvictionFunc(FinalizeEvictionFunc && func) { finalize_eviction_func = func; } + private: struct KeyCandidates { @@ -28,6 +33,8 @@ private: std::unordered_map candidates; size_t candidates_size = 0; + std::vector invalidated_queue_entries; + FinalizeEvictionFunc finalize_eviction_func; }; using EvictionCandidatesPtr = std::unique_ptr; diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 9c705ddc27c..90508d74554 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -814,12 +814,10 @@ bool FileCache::tryReserve( } EvictionCandidates eviction_candidates; - IFileCachePriority::FinalizeEvictionFunc finalize_eviction_func; if (query_priority) { - if (!query_priority->collectCandidatesForEviction( - size, reserve_stat, eviction_candidates, {}, finalize_eviction_func, user.user_id, cache_lock)) + if (!query_priority->collectCandidatesForEviction(size, reserve_stat, eviction_candidates, {}, user.user_id, cache_lock)) return false; LOG_TEST(log, "Query limits satisfied (while reserving for {}:{})", file_segment.key(), file_segment.offset()); @@ -832,26 +830,38 @@ bool FileCache::tryReserve( auto queue_iterator = file_segment.getQueueIterator(); chassert(!queue_iterator || file_segment.getReservedSize() > 0); - if (!main_priority->collectCandidatesForEviction( - size, reserve_stat, eviction_candidates, queue_iterator, finalize_eviction_func, user.user_id, cache_lock)) + if (!main_priority->collectCandidatesForEviction(size, reserve_stat, eviction_candidates, queue_iterator, user.user_id, cache_lock)) return false; + /// Let's release cache lock if we are going to remove files from filesystem. + bool release_lock = eviction_candidates.size() > 0; + if (release_lock) + cache_lock.unlock(); + if (!file_segment.getKeyMetadata()->createBaseDirectory()) return false; - eviction_candidates.evict(query_context.get(), cache_lock); + /// Remove eviction candidates from filesystem. + eviction_candidates.evict(); - if (finalize_eviction_func) - finalize_eviction_func(cache_lock); + /// Take cache lock again. + if (release_lock) + cache_lock.lock(); + /// Remove invalidated queue entries and execute (only for SLRU) finalize func. + eviction_candidates.finalize(query_context.get(), cache_lock); + + /// Space reservation is incremental, so file_segment_metadata is created first (with state Empty), + /// and queue_iterator is assigned on first space reservation attempt + /// (so it is nullptr here if we are reserving for the first time). if (queue_iterator) { - queue_iterator->updateSize(size); + /// Increase size of queue entry. + queue_iterator->incrementSize(size, cache_lock); } else { - /// Space reservation is incremental, so file_segment_metadata is created first (with state empty), - /// and getQueueIterator() is assigned on first space reservation attempt. + /// Create a new queue entry and assign currently reserved size to it. queue_iterator = main_priority->add(file_segment.getKeyMetadata(), file_segment.offset(), size, user, cache_lock); file_segment.setQueueIterator(queue_iterator); } @@ -863,7 +873,7 @@ bool FileCache::tryReserve( { auto query_queue_it = query_context->tryGet(file_segment.key(), file_segment.offset(), cache_lock); if (query_queue_it) - query_queue_it->updateSize(size); + query_queue_it->incrementSize(size, cache_lock); else query_context->add(file_segment.getKeyMetadata(), file_segment.offset(), size, user, cache_lock); } diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index bc036166940..37270cf0873 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -45,7 +45,12 @@ public: virtual size_t increasePriority(const CacheGuard::Lock &) = 0; - virtual void updateSize(int64_t size) = 0; + /// Note: IncrementSize unlike decrementSize requires a cache lock, because + /// it requires more consistency guarantees for eviction. + + virtual void incrementSize(size_t size, const CacheGuard::Lock &) = 0; + + virtual void decrementSize(size_t size) = 0; virtual void remove(const CacheGuard::Lock &) = 0; @@ -93,13 +98,11 @@ public: virtual PriorityDumpPtr dump(const CacheGuard::Lock &) = 0; - using FinalizeEvictionFunc = std::function; virtual bool collectCandidatesForEviction( size_t size, FileCacheReserveStat & stat, EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, - FinalizeEvictionFunc & finalize_eviction_func, const UserID & user_id, const CacheGuard::Lock &) = 0; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index bce03b60024..05bbc26e602 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -15,6 +15,7 @@ namespace CurrentMetrics namespace ProfileEvents { extern const Event FilesystemCacheEvictionSkippedFileSegments; + extern const Event FilesystemCacheEvictionSkippedEvictingFileSegments; extern const Event FilesystemCacheEvictionTries; extern const Event FilesystemCacheEvictMicroseconds; extern const Event FilesystemCacheEvictedBytes; @@ -223,7 +224,6 @@ bool LRUFileCachePriority::collectCandidatesForEviction( FileCacheReserveStat & stat, EvictionCandidates & res, IFileCachePriority::IteratorPtr, - FinalizeEvictionFunc &, const UserID &, const CacheGuard::Lock & lock) { @@ -237,7 +237,11 @@ bool LRUFileCachePriority::collectCandidatesForEviction( const auto & file_segment = segment_metadata->file_segment; chassert(file_segment->assertCorrectness()); - if (segment_metadata->releasable()) + if (segment_metadata->evicting()) + { + ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictionSkippedEvictingFileSegments); + } + else if (segment_metadata->releasable()) { res.add(locked_key, segment_metadata); stat.update(segment_metadata->size(), file_segment->getKind(), true); @@ -375,20 +379,34 @@ void LRUFileCachePriority::LRUIterator::invalidate() entry->size = 0; } -void LRUFileCachePriority::LRUIterator::updateSize(int64_t size) +void LRUFileCachePriority::LRUIterator::incrementSize(size_t size, const CacheGuard::Lock &) { assertValid(); const auto & entry = *iterator; LOG_TEST( cache_priority->log, - "Update size with {} in LRU queue for key: {}, offset: {}, previous size: {}", + "Increment size with {} in LRU queue for key: {}, offset: {}, previous size: {}", size, entry->key, entry->offset, entry->size); cache_priority->updateSize(size); entry->size += size; } +void LRUFileCachePriority::LRUIterator::decrementSize(size_t size) +{ + assertValid(); + + const auto & entry = *iterator; + LOG_TEST( + cache_priority->log, + "Decrement size with {} in LRU queue for key: {}, offset: {}, previous size: {}", + size, entry->key, entry->offset, entry->size); + + cache_priority->updateSize(-size); + entry->size -= size; +} + size_t LRUFileCachePriority::LRUIterator::increasePriority(const CacheGuard::Lock &) { assertValid(); diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index a74a4b8b621..d8907f678a2 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -47,7 +47,6 @@ public: FileCacheReserveStat & stat, EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, - FinalizeEvictionFunc & finalize_eviction_func, const UserID & user_id, const CacheGuard::Lock &) override; @@ -114,7 +113,9 @@ public: void invalidate() override; - void updateSize(int64_t size) override; + void incrementSize(size_t size, const CacheGuard::Lock &) override; + + void decrementSize(size_t size) override; QueueEntryType getType() const override { return QueueEntryType::LRU; } diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 727f2762cca..eb3b4e44c8c 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -1012,7 +1012,7 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( file_segment->cache, key_metadata, file_segment->queue_iterator); if (diff) - metadata->getQueueIterator()->updateSize(-diff); + metadata->getQueueIterator()->decrementSize(diff); chassert(file_segment->assertCorrectnessUnlocked(segment_lock)); } diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 43f1c1012ba..543d6a03669 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -101,7 +101,6 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( FileCacheReserveStat & stat, EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, - FinalizeEvictionFunc & finalize_eviction_func, const UserID & user_id, const CacheGuard::Lock & lock) { @@ -109,7 +108,7 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( /// for a corresponding file segment, so it will be directly put into probationary queue. if (!reservee) { - return probationary_queue.collectCandidatesForEviction(size, stat, res, reservee, finalize_eviction_func, user_id, lock); + return probationary_queue.collectCandidatesForEviction(size, stat, res, reservee, user_id, lock); } /// If `it` not nullptr (e.g. is already in some queue), @@ -117,7 +116,7 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( /// (in order to know where we need to free space). if (!assert_cast(reservee.get())->is_protected) { - return probationary_queue.collectCandidatesForEviction(size, stat, res, reservee, finalize_eviction_func, user_id, lock); + return probationary_queue.collectCandidatesForEviction(size, stat, res, reservee, user_id, lock); } /// Entry is in protected queue. @@ -132,18 +131,17 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( /// required to make space for additionary `size` bytes for entry. auto downgrade_candidates = std::make_shared(); FileCacheReserveStat downgrade_stat; - FinalizeEvictionFunc noop; - if (!protected_queue.collectCandidatesForEviction(size, downgrade_stat, *downgrade_candidates, reservee, noop, user_id, lock)) + if (!protected_queue.collectCandidatesForEviction(size, downgrade_stat, *downgrade_candidates, reservee, user_id, lock)) return false; const size_t size_to_downgrade = downgrade_stat.stat.releasable_size; if (!probationary_queue.canFit(size_to_downgrade, lock) - && !probationary_queue.collectCandidatesForEviction(size_to_downgrade, stat, res, reservee, noop, user_id, lock)) + && !probationary_queue.collectCandidatesForEviction(size_to_downgrade, stat, res, reservee, user_id, lock)) return false; - finalize_eviction_func = [=, this](const CacheGuard::Lock & lk) mutable + res.setFinalizeEvictionFunc([=, this](const CacheGuard::Lock & lk) mutable { for (const auto & [key, key_candidates] : *downgrade_candidates) { @@ -154,7 +152,7 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( candidate_it->is_protected = false; } } - }; + }); return true; } @@ -186,9 +184,8 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach /// queue to probationary queue. EvictionCandidates downgrade_candidates; FileCacheReserveStat downgrade_stat; - FinalizeEvictionFunc noop; - if (!protected_queue.collectCandidatesForEviction(size, downgrade_stat, downgrade_candidates, {}, noop, "", lock)) + if (!protected_queue.collectCandidatesForEviction(size, downgrade_stat, downgrade_candidates, {}, "", lock)) { /// We cannot make space for entry to be moved to protected queue /// (not enough releasable file segments). @@ -211,7 +208,7 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach if (size_to_free) { - if (!probationary_queue.collectCandidatesForEviction(size_to_free, stat, eviction_candidates, {}, noop, {}, lock)) + if (!probationary_queue.collectCandidatesForEviction(size_to_free, stat, eviction_candidates, {}, {}, lock)) { /// "downgrade" candidates cannot be moved to probationary queue, /// so entry cannot be moved to protected queue as well. @@ -220,7 +217,7 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach return; } /// Make space for "downgrade" candidates. - eviction_candidates.evict(nullptr, lock); + eviction_candidates.evict(); } /// All checks passed, now we can move downgrade candidates to @@ -294,10 +291,16 @@ size_t SLRUFileCachePriority::SLRUIterator::increasePriority(const CacheGuard::L return getEntry()->hits; } -void SLRUFileCachePriority::SLRUIterator::updateSize(int64_t size) +void SLRUFileCachePriority::SLRUIterator::incrementSize(size_t size, const CacheGuard::Lock & lock) { assertValid(); - lru_iterator.updateSize(size); + lru_iterator.incrementSize(size, lock); +} + +void SLRUFileCachePriority::SLRUIterator::decrementSize(size_t size) +{ + assertValid(); + lru_iterator.decrementSize(size); } void SLRUFileCachePriority::SLRUIterator::invalidate() diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h index d97fa80a6c7..28e61396572 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.h +++ b/src/Interpreters/Cache/SLRUFileCachePriority.h @@ -44,7 +44,6 @@ public: FileCacheReserveStat & stat, EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, - FinalizeEvictionFunc & finalize_eviction_func, const UserID & user_id, const CacheGuard::Lock &) override; @@ -80,7 +79,9 @@ public: void invalidate() override; - void updateSize(int64_t size) override; + void incrementSize(size_t size, const CacheGuard::Lock &) override; + + void decrementSize(size_t size) override; QueueEntryType getType() const override { return is_protected ? QueueEntryType::SLRU_Protected : QueueEntryType::SLRU_Probationary; } From db48b9749b8a9ebb8500c52b32ca0bfa61481df1 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 11 Mar 2024 15:35:17 +0000 Subject: [PATCH 0100/1165] better _part_offset column --- .../MergeTree/MergeTreeRangeReader.cpp | 8 +- src/Storages/MergeTree/MergeTreeRangeReader.h | 2 + .../02890_describe_table_options.reference | 256 ++++++++++-------- 3 files changed, 144 insertions(+), 122 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index bbc4adaf458..eb757e1d8c7 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -1146,6 +1146,12 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t if (!result.rows_per_granule.empty()) result.adjustLastGranule(); + fillVirtualColumns(result, leading_begin_part_offset, leading_end_part_offset); + return result; +} + +void MergeTreeRangeReader::fillVirtualColumns(ReadResult & result, UInt64 leading_begin_part_offset, UInt64 leading_end_part_offset) +{ ColumnPtr part_offset_column; auto add_offset_column = [&](const auto & column_name) @@ -1169,8 +1175,6 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t /// Column _block_offset is the same as _part_offset if it's not persisted in part. if (read_sample_block.has(BlockOffsetColumn::name)) add_offset_column(BlockOffsetColumn::name); - - return result; } ColumnPtr MergeTreeRangeReader::createPartOffsetColumn(ReadResult & result, UInt64 leading_begin_part_offset, UInt64 leading_end_part_offset) diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.h b/src/Storages/MergeTree/MergeTreeRangeReader.h index 51fbbc8b052..b282ada6038 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.h +++ b/src/Storages/MergeTree/MergeTreeRangeReader.h @@ -311,6 +311,8 @@ private: ReadResult startReadingChain(size_t max_rows, MarkRanges & ranges); Columns continueReadingChain(const ReadResult & result, size_t & num_rows); void executePrewhereActionsAndFilterColumns(ReadResult & result) const; + + void fillVirtualColumns(ReadResult & result, UInt64 leading_begin_part_offset, UInt64 leading_end_part_offset); ColumnPtr createPartOffsetColumn(ReadResult & result, UInt64 leading_begin_part_offset, UInt64 leading_end_part_offset); IMergeTreeReader * merge_tree_reader = nullptr; diff --git a/tests/queries/0_stateless/02890_describe_table_options.reference b/tests/queries/0_stateless/02890_describe_table_options.reference index ff58202ae49..fe73e677432 100644 --- a/tests/queries/0_stateless/02890_describe_table_options.reference +++ b/tests/queries/0_stateless/02890_describe_table_options.reference @@ -34,70 +34,78 @@ DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCom └───────────┴───────────────────────────┴──────────────┴────────────────────┴──────────────┴──────────────────┴────────────────┴──────────────┘ SET describe_compact_output = 0, describe_include_virtual_columns = 1, describe_include_subcolumns = 0; DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_virtual─┐ -│ id │ UInt64 │ │ │ index column │ │ │ 0 │ -│ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ -│ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ -│ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 1 │ -│ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 1 │ -│ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 1 │ -│ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 1 │ -│ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 1 │ -│ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 1 │ -│ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 1 │ -│ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 1 │ -└────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴────────────┘ +┌─name───────────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_virtual─┐ +│ id │ UInt64 │ │ │ index column │ │ │ 0 │ +│ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ +│ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ +│ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 1 │ +│ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 1 │ +│ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 1 │ +│ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 1 │ +│ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 1 │ +│ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 1 │ +│ _part_data_version │ UInt64 │ │ │ Data version of part (either min block number or mutation version) │ │ │ 1 │ +│ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 1 │ +│ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 1 │ +│ _block_offset │ UInt64 │ │ │ Persisted original number of row in block that was assigned at insert │ Delta, LZ4 │ │ 1 │ +└────────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴────────────┘ DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; -┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_virtual─┐ -│ id │ UInt64 │ │ │ index column │ │ │ 0 │ -│ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ -│ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ -│ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 1 │ -│ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 1 │ -│ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 1 │ -│ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 1 │ -│ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 1 │ -│ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 1 │ -│ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 1 │ -│ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 1 │ -│ _shard_num │ UInt32 │ │ │ Deprecated. Use function shardNum instead │ │ │ 1 │ -└────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴────────────┘ +┌─name───────────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_virtual─┐ +│ id │ UInt64 │ │ │ index column │ │ │ 0 │ +│ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ +│ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ +│ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 1 │ +│ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 1 │ +│ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 1 │ +│ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 1 │ +│ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 1 │ +│ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 1 │ +│ _part_data_version │ UInt64 │ │ │ Data version of part (either min block number or mutation version) │ │ │ 1 │ +│ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 1 │ +│ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 1 │ +│ _block_offset │ UInt64 │ │ │ Persisted original number of row in block that was assigned at insert │ Delta, LZ4 │ │ 1 │ +│ _shard_num │ UInt32 │ │ │ Deprecated. Use function shardNum instead │ │ │ 1 │ +└────────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴────────────┘ SET describe_compact_output = 0, describe_include_virtual_columns = 1, describe_include_subcolumns = 1; DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┬─is_virtual─┐ -│ id │ UInt64 │ │ │ index column │ │ │ 0 │ 0 │ -│ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ 0 │ -│ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ 0 │ -│ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 0 │ 1 │ -│ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 0 │ 1 │ -│ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 0 │ 1 │ -│ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 0 │ 1 │ -│ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 0 │ 1 │ -│ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 0 │ 1 │ -│ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 0 │ 1 │ -│ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 0 │ 1 │ -│ arr.size0 │ UInt64 │ │ │ │ │ │ 1 │ 0 │ -│ t.a │ String │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ -│ t.b │ UInt64 │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ -└────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴──────────────┴────────────┘ +┌─name───────────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┬─is_virtual─┐ +│ id │ UInt64 │ │ │ index column │ │ │ 0 │ 0 │ +│ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ 0 │ +│ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ 0 │ +│ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 0 │ 1 │ +│ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 0 │ 1 │ +│ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 0 │ 1 │ +│ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 0 │ 1 │ +│ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 0 │ 1 │ +│ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 0 │ 1 │ +│ _part_data_version │ UInt64 │ │ │ Data version of part (either min block number or mutation version) │ │ │ 0 │ 1 │ +│ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 0 │ 1 │ +│ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 0 │ 1 │ +│ _block_offset │ UInt64 │ │ │ Persisted original number of row in block that was assigned at insert │ Delta, LZ4 │ │ 0 │ 1 │ +│ arr.size0 │ UInt64 │ │ │ │ │ │ 1 │ 0 │ +│ t.a │ String │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ +│ t.b │ UInt64 │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ +└────────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴──────────────┴────────────┘ DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; -┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┬─is_virtual─┐ -│ id │ UInt64 │ │ │ index column │ │ │ 0 │ 0 │ -│ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ 0 │ -│ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ 0 │ -│ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 0 │ 1 │ -│ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 0 │ 1 │ -│ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 0 │ 1 │ -│ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 0 │ 1 │ -│ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 0 │ 1 │ -│ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 0 │ 1 │ -│ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 0 │ 1 │ -│ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 0 │ 1 │ -│ _shard_num │ UInt32 │ │ │ Deprecated. Use function shardNum instead │ │ │ 0 │ 1 │ -│ arr.size0 │ UInt64 │ │ │ │ │ │ 1 │ 0 │ -│ t.a │ String │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ -│ t.b │ UInt64 │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ -└────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴──────────────┴────────────┘ +┌─name───────────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┬─is_virtual─┐ +│ id │ UInt64 │ │ │ index column │ │ │ 0 │ 0 │ +│ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ 0 │ +│ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ 0 │ +│ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 0 │ 1 │ +│ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 0 │ 1 │ +│ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 0 │ 1 │ +│ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 0 │ 1 │ +│ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 0 │ 1 │ +│ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 0 │ 1 │ +│ _part_data_version │ UInt64 │ │ │ Data version of part (either min block number or mutation version) │ │ │ 0 │ 1 │ +│ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 0 │ 1 │ +│ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 0 │ 1 │ +│ _block_offset │ UInt64 │ │ │ Persisted original number of row in block that was assigned at insert │ Delta, LZ4 │ │ 0 │ 1 │ +│ _shard_num │ UInt32 │ │ │ Deprecated. Use function shardNum instead │ │ │ 0 │ 1 │ +│ arr.size0 │ UInt64 │ │ │ │ │ │ 1 │ 0 │ +│ t.a │ String │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ +│ t.b │ UInt64 │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ +└────────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴──────────────┴────────────┘ SET describe_compact_output = 1, describe_include_virtual_columns = 0, describe_include_subcolumns = 0; DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; ┌─name─┬─type──────────────────────┐ @@ -132,67 +140,75 @@ DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCom └───────────┴───────────────────────────┴──────────────┘ SET describe_compact_output = 1, describe_include_virtual_columns = 1, describe_include_subcolumns = 0; DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -┌─name───────────┬─type──────────────────────┬─is_virtual─┐ -│ id │ UInt64 │ 0 │ -│ arr │ Array(UInt64) │ 0 │ -│ t │ Tuple(a String, b UInt64) │ 0 │ -│ _part │ LowCardinality(String) │ 1 │ -│ _part_index │ UInt64 │ 1 │ -│ _part_uuid │ UUID │ 1 │ -│ _partition_id │ LowCardinality(String) │ 1 │ -│ _sample_factor │ Float64 │ 1 │ -│ _part_offset │ UInt64 │ 1 │ -│ _row_exists │ UInt8 │ 1 │ -│ _block_number │ UInt64 │ 1 │ -└────────────────┴───────────────────────────┴────────────┘ +┌─name───────────────┬─type──────────────────────┬─is_virtual─┐ +│ id │ UInt64 │ 0 │ +│ arr │ Array(UInt64) │ 0 │ +│ t │ Tuple(a String, b UInt64) │ 0 │ +│ _part │ LowCardinality(String) │ 1 │ +│ _part_index │ UInt64 │ 1 │ +│ _part_uuid │ UUID │ 1 │ +│ _partition_id │ LowCardinality(String) │ 1 │ +│ _sample_factor │ Float64 │ 1 │ +│ _part_offset │ UInt64 │ 1 │ +│ _part_data_version │ UInt64 │ 1 │ +│ _row_exists │ UInt8 │ 1 │ +│ _block_number │ UInt64 │ 1 │ +│ _block_offset │ UInt64 │ 1 │ +└────────────────────┴───────────────────────────┴────────────┘ DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; -┌─name───────────┬─type──────────────────────┬─is_virtual─┐ -│ id │ UInt64 │ 0 │ -│ arr │ Array(UInt64) │ 0 │ -│ t │ Tuple(a String, b UInt64) │ 0 │ -│ _part │ LowCardinality(String) │ 1 │ -│ _part_index │ UInt64 │ 1 │ -│ _part_uuid │ UUID │ 1 │ -│ _partition_id │ LowCardinality(String) │ 1 │ -│ _sample_factor │ Float64 │ 1 │ -│ _part_offset │ UInt64 │ 1 │ -│ _row_exists │ UInt8 │ 1 │ -│ _block_number │ UInt64 │ 1 │ -│ _shard_num │ UInt32 │ 1 │ -└────────────────┴───────────────────────────┴────────────┘ +┌─name───────────────┬─type──────────────────────┬─is_virtual─┐ +│ id │ UInt64 │ 0 │ +│ arr │ Array(UInt64) │ 0 │ +│ t │ Tuple(a String, b UInt64) │ 0 │ +│ _part │ LowCardinality(String) │ 1 │ +│ _part_index │ UInt64 │ 1 │ +│ _part_uuid │ UUID │ 1 │ +│ _partition_id │ LowCardinality(String) │ 1 │ +│ _sample_factor │ Float64 │ 1 │ +│ _part_offset │ UInt64 │ 1 │ +│ _part_data_version │ UInt64 │ 1 │ +│ _row_exists │ UInt8 │ 1 │ +│ _block_number │ UInt64 │ 1 │ +│ _block_offset │ UInt64 │ 1 │ +│ _shard_num │ UInt32 │ 1 │ +└────────────────────┴───────────────────────────┴────────────┘ SET describe_compact_output = 1, describe_include_virtual_columns = 1, describe_include_subcolumns = 1; DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -┌─name───────────┬─type──────────────────────┬─is_subcolumn─┬─is_virtual─┐ -│ id │ UInt64 │ 0 │ 0 │ -│ arr │ Array(UInt64) │ 0 │ 0 │ -│ t │ Tuple(a String, b UInt64) │ 0 │ 0 │ -│ _part │ LowCardinality(String) │ 0 │ 1 │ -│ _part_index │ UInt64 │ 0 │ 1 │ -│ _part_uuid │ UUID │ 0 │ 1 │ -│ _partition_id │ LowCardinality(String) │ 0 │ 1 │ -│ _sample_factor │ Float64 │ 0 │ 1 │ -│ _part_offset │ UInt64 │ 0 │ 1 │ -│ _row_exists │ UInt8 │ 0 │ 1 │ -│ _block_number │ UInt64 │ 0 │ 1 │ -│ arr.size0 │ UInt64 │ 1 │ 0 │ -│ t.a │ String │ 1 │ 0 │ -│ t.b │ UInt64 │ 1 │ 0 │ -└────────────────┴───────────────────────────┴──────────────┴────────────┘ +┌─name───────────────┬─type──────────────────────┬─is_subcolumn─┬─is_virtual─┐ +│ id │ UInt64 │ 0 │ 0 │ +│ arr │ Array(UInt64) │ 0 │ 0 │ +│ t │ Tuple(a String, b UInt64) │ 0 │ 0 │ +│ _part │ LowCardinality(String) │ 0 │ 1 │ +│ _part_index │ UInt64 │ 0 │ 1 │ +│ _part_uuid │ UUID │ 0 │ 1 │ +│ _partition_id │ LowCardinality(String) │ 0 │ 1 │ +│ _sample_factor │ Float64 │ 0 │ 1 │ +│ _part_offset │ UInt64 │ 0 │ 1 │ +│ _part_data_version │ UInt64 │ 0 │ 1 │ +│ _row_exists │ UInt8 │ 0 │ 1 │ +│ _block_number │ UInt64 │ 0 │ 1 │ +│ _block_offset │ UInt64 │ 0 │ 1 │ +│ arr.size0 │ UInt64 │ 1 │ 0 │ +│ t.a │ String │ 1 │ 0 │ +│ t.b │ UInt64 │ 1 │ 0 │ +└────────────────────┴───────────────────────────┴──────────────┴────────────┘ DESCRIBE remote(default, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; -┌─name───────────┬─type──────────────────────┬─is_subcolumn─┬─is_virtual─┐ -│ id │ UInt64 │ 0 │ 0 │ -│ arr │ Array(UInt64) │ 0 │ 0 │ -│ t │ Tuple(a String, b UInt64) │ 0 │ 0 │ -│ _part │ LowCardinality(String) │ 0 │ 1 │ -│ _part_index │ UInt64 │ 0 │ 1 │ -│ _part_uuid │ UUID │ 0 │ 1 │ -│ _partition_id │ LowCardinality(String) │ 0 │ 1 │ -│ _sample_factor │ Float64 │ 0 │ 1 │ -│ _part_offset │ UInt64 │ 0 │ 1 │ -│ _row_exists │ UInt8 │ 0 │ 1 │ -│ _block_number │ UInt64 │ 0 │ 1 │ -│ _shard_num │ UInt32 │ 0 │ 1 │ -│ arr.size0 │ UInt64 │ 1 │ 0 │ -│ t.a │ String │ 1 │ 0 │ -│ t.b │ UInt64 │ 1 │ 0 │ -└────────────────┴───────────────────────────┴──────────────┴────────────┘ +┌─name───────────────┬─type──────────────────────┬─is_subcolumn─┬─is_virtual─┐ +│ id │ UInt64 │ 0 │ 0 │ +│ arr │ Array(UInt64) │ 0 │ 0 │ +│ t │ Tuple(a String, b UInt64) │ 0 │ 0 │ +│ _part │ LowCardinality(String) │ 0 │ 1 │ +│ _part_index │ UInt64 │ 0 │ 1 │ +│ _part_uuid │ UUID │ 0 │ 1 │ +│ _partition_id │ LowCardinality(String) │ 0 │ 1 │ +│ _sample_factor │ Float64 │ 0 │ 1 │ +│ _part_offset │ UInt64 │ 0 │ 1 │ +│ _part_data_version │ UInt64 │ 0 │ 1 │ +│ _row_exists │ UInt8 │ 0 │ 1 │ +│ _block_number │ UInt64 │ 0 │ 1 │ +│ _block_offset │ UInt64 │ 0 │ 1 │ +│ _shard_num │ UInt32 │ 0 │ 1 │ +│ arr.size0 │ UInt64 │ 1 │ 0 │ +│ t.a │ String │ 1 │ 0 │ +│ t.b │ UInt64 │ 1 │ 0 │ +└────────────────────┴───────────────────────────┴──────────────┴────────────┘ From 5608005b34515cc1b12b6043377cb2c78683c07b Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 11 Mar 2024 21:22:45 +0100 Subject: [PATCH 0101/1165] Revert "Revert "Don't allow to set max_parallel_replicas to 0 as it doesn't make sense"" --- src/Client/ConnectionPoolWithFailover.cpp | 10 ++++++++++ src/Client/HedgedConnectionsFactory.cpp | 6 +++++- src/Client/HedgedConnectionsFactory.h | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Planner/PlannerJoinTree.cpp | 4 ++-- .../03001_max_parallel_replicas_zero_value.reference | 0 .../03001_max_parallel_replicas_zero_value.sql | 5 +++++ 7 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.reference create mode 100644 tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 492fd4ae9e2..ad8ed0067d8 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int ALL_CONNECTION_TRIES_FAILED; + extern const int BAD_ARGUMENTS; } @@ -191,11 +192,20 @@ std::vector ConnectionPoolWithFailover::g max_entries = nested_pools.size(); } else if (pool_mode == PoolMode::GET_ONE) + { max_entries = 1; + } else if (pool_mode == PoolMode::GET_MANY) + { + if (settings.max_parallel_replicas == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of the setting max_parallel_replicas must be greater than 0"); + max_entries = settings.max_parallel_replicas; + } else + { throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown pool allocation mode"); + } if (!priority_func) priority_func = makeGetPriorityFunc(settings); diff --git a/src/Client/HedgedConnectionsFactory.cpp b/src/Client/HedgedConnectionsFactory.cpp index f5b074a0257..703cc1f8821 100644 --- a/src/Client/HedgedConnectionsFactory.cpp +++ b/src/Client/HedgedConnectionsFactory.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes extern const int ALL_CONNECTION_TRIES_FAILED; extern const int ALL_REPLICAS_ARE_STALE; extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } HedgedConnectionsFactory::HedgedConnectionsFactory( @@ -82,7 +83,10 @@ std::vector HedgedConnectionsFactory::getManyConnections(PoolMode } case PoolMode::GET_MANY: { - max_entries = max_parallel_replicas; + if (max_parallel_replicas == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of the setting max_parallel_replicas must be greater than 0"); + + max_entries = std::min(max_parallel_replicas, shuffled_pools.size()); break; } } diff --git a/src/Client/HedgedConnectionsFactory.h b/src/Client/HedgedConnectionsFactory.h index ce7b553acdd..dd600d58e1e 100644 --- a/src/Client/HedgedConnectionsFactory.h +++ b/src/Client/HedgedConnectionsFactory.h @@ -158,7 +158,7 @@ private: /// checking the number of requested replicas that are still in process). size_t requested_connections_count = 0; - const size_t max_parallel_replicas = 0; + const size_t max_parallel_replicas = 1; const bool skip_unavailable_shards = 0; }; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index bcedba7346d..e28d8366aa7 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -947,7 +947,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() if (number_of_replicas_to_use <= 1) { context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); - context->setSetting("max_parallel_replicas", UInt64{0}); + context->setSetting("max_parallel_replicas", UInt64{1}); LOG_DEBUG(log, "Disabling parallel replicas because there aren't enough rows to read"); return true; } diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 7b3fb0c5c91..0fe943e0bc7 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -295,7 +295,7 @@ bool applyTrivialCountIfPossible( /// The query could use trivial count if it didn't use parallel replicas, so let's disable it query_context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); - query_context->setSetting("max_parallel_replicas", UInt64{0}); + query_context->setSetting("max_parallel_replicas", UInt64{1}); LOG_TRACE(getLogger("Planner"), "Disabling parallel replicas to be able to use a trivial count optimization"); } @@ -756,7 +756,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres { planner_context->getMutableQueryContext()->setSetting( "allow_experimental_parallel_reading_from_replicas", Field(0)); - planner_context->getMutableQueryContext()->setSetting("max_parallel_replicas", UInt64{0}); + planner_context->getMutableQueryContext()->setSetting("max_parallel_replicas", UInt64{1}); LOG_DEBUG(getLogger("Planner"), "Disabling parallel replicas because there aren't enough rows to read"); } else if (number_of_replicas_to_use < settings.max_parallel_replicas) diff --git a/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.reference b/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql b/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql new file mode 100644 index 00000000000..611aa4777ba --- /dev/null +++ b/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql @@ -0,0 +1,5 @@ +drop table if exists test_d; +create table test_d engine=Distributed(test_cluster_two_shard_three_replicas_localhost, system, numbers); +select * from test_d limit 10 settings max_parallel_replicas = 0, prefer_localhost_replica = 0; --{serverError BAD_ARGUMENTS} +drop table test_d; + From 29fce4143d1f177efdf1864d41429cfadea22ff1 Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 12 Mar 2024 12:07:24 +0800 Subject: [PATCH 0102/1165] [fix] log level from fatal->error when hardlink and copy both fail --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c76ffeee874..97968f1b9c1 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7174,7 +7174,7 @@ std::pair MergeTreeData::cloneAn } } if (!copy_successful) - LOG_FATAL(&Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail"); + LOG_ERROR(&Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail"); } From 10c7ea7a29d8426fcf4d0ca09c778cdd3e56fbbd Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 12 Mar 2024 14:32:07 +0800 Subject: [PATCH 0103/1165] [debug] fast test again --- src/Storages/StorageMergeTree.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 055a48ad998..928ee094583 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2083,7 +2083,6 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con MergeTreePartInfo dst_part_info(partition_id, temp_index, temp_index, src_part->info.level); IDataPartStorage::ClonePartParams clone_params{.txn = local_context->getCurrentTransaction()}; - bool on_same_disk = false; for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) if (disk->getName() == src_part->getDataPartStorage().getDiskName()) From c1b94b2170acbf72d066928fd168c18dc571d505 Mon Sep 17 00:00:00 2001 From: unashi Date: Tue, 12 Mar 2024 15:33:43 +0800 Subject: [PATCH 0104/1165] [debug] fast test again again --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 97968f1b9c1..7fc504d71f1 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7174,7 +7174,7 @@ std::pair MergeTreeData::cloneAn } } if (!copy_successful) - LOG_ERROR(&Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail"); + LOG_ERROR(&Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail."); } From db3638ef21b838636a3bb4a607c83b17374067a5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Mar 2024 13:57:07 +0100 Subject: [PATCH 0105/1165] Enable processors profiling by default It is only 1 record per processor for each query, while there are way more less columns then in query_log, so it is pretty light weight. Signed-off-by: Azat Khuzhin --- src/Core/Settings.h | 2 +- src/Core/SettingsChangesHistory.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d70a6cf51c5..3275a3f91ef 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -263,7 +263,7 @@ class IColumn; M(UInt64, log_queries_cut_to_length, 100000, "If query length is greater than specified threshold (in bytes), then cut query when writing to query log. Also limit length of printed query in ordinary text log.", 0) \ M(Float, log_queries_probability, 1., "Log queries with the specified probability.", 0) \ \ - M(Bool, log_processors_profiles, false, "Log Processors profile events.", 0) \ + M(Bool, log_processors_profiles, true, "Log Processors profile events.", 0) \ M(DistributedProductMode, distributed_product_mode, DistributedProductMode::DENY, "How are distributed subqueries performed inside IN or JOIN sections?", IMPORTANT) \ \ M(UInt64, max_concurrent_queries_for_all_users, 0, "The maximum number of concurrent requests for all users.", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index e680c02671a..144affd1b36 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -93,6 +93,7 @@ static std::map sett {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"}, {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."}, {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, + {"log_processors_profiles", false, true, "Enable by default"}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, From defed923313e2cf8c33d3b0890d6a2b86e563c45 Mon Sep 17 00:00:00 2001 From: serxa Date: Tue, 12 Mar 2024 11:38:27 +0000 Subject: [PATCH 0106/1165] do nothing in `waitForOutdatedPartsToBeLoaded()` if loading is not required --- src/Storages/MergeTree/MergeTreeData.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index d56cf761cf4..85389828e57 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1979,6 +1979,15 @@ void MergeTreeData::waitForOutdatedPartsToBeLoaded() const TSA_NO_THREAD_SAFETY_ if (isStaticStorage()) return; + /// If waiting is not required, do NOT log and do NOT enable/disable turbo mode to make `waitForOutdatedPartsToBeLoaded` a lightweight check + { + std::unique_lock lock(outdated_data_parts_mutex); + if (outdated_data_parts_loading_canceled) + throw Exception(ErrorCodes::NOT_INITIALIZED, "Loading of outdated data parts was already canceled"); + if (outdated_data_parts_loading_finished) + return; + } + /// We need to load parts as fast as possible getOutdatedPartsLoadingThreadPool().enableTurboMode(); SCOPE_EXIT({ From dd6599868adb6cbc3306a5946cae4ee3f833c138 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 12 Mar 2024 12:06:25 +0000 Subject: [PATCH 0107/1165] Better check for 0 setting value --- src/Client/ConnectionPoolWithFailover.cpp | 3 -- src/Client/HedgedConnectionsFactory.cpp | 3 -- src/Core/Settings.h | 2 +- src/Core/SettingsFields.cpp | 36 +++++++++++++++++++++++ src/Core/SettingsFields.h | 15 ++++++++++ 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index ad8ed0067d8..94531f58bc6 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -197,9 +197,6 @@ std::vector ConnectionPoolWithFailover::g } else if (pool_mode == PoolMode::GET_MANY) { - if (settings.max_parallel_replicas == 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of the setting max_parallel_replicas must be greater than 0"); - max_entries = settings.max_parallel_replicas; } else diff --git a/src/Client/HedgedConnectionsFactory.cpp b/src/Client/HedgedConnectionsFactory.cpp index 703cc1f8821..6b22cc18674 100644 --- a/src/Client/HedgedConnectionsFactory.cpp +++ b/src/Client/HedgedConnectionsFactory.cpp @@ -83,9 +83,6 @@ std::vector HedgedConnectionsFactory::getManyConnections(PoolMode } case PoolMode::GET_MANY: { - if (max_parallel_replicas == 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of the setting max_parallel_replicas must be greater than 0"); - max_entries = std::min(max_parallel_replicas, shuffled_pools.size()); break; } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d70a6cf51c5..b23538cf209 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -178,7 +178,7 @@ class IColumn; \ M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \ \ - M(UInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled.", 0) \ + M(NonZeroUInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled. Should be always greater than 0", 0) \ M(UInt64, parallel_replicas_count, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the number of parallel replicas participating in query processing.", 0) \ M(UInt64, parallel_replica_offset, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the index of the replica participating in query processing among parallel replicas.", 0) \ M(String, parallel_replicas_custom_key, "", "Custom key assigning work to replicas when parallel replicas are used.", 0) \ diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 001d3e09dc9..caa8b3fdffd 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -575,4 +575,40 @@ void SettingFieldCustom::readBinary(ReadBuffer & in) parseFromString(str); } +SettingFieldNonZeroUInt64::SettingFieldNonZeroUInt64(UInt64 x) : SettingFieldUInt64(x) +{ + checkValueNonZero(); +} + +SettingFieldNonZeroUInt64::SettingFieldNonZeroUInt64(const DB::Field & f) : SettingFieldUInt64(f) +{ + checkValueNonZero(); +} + +SettingFieldNonZeroUInt64 & SettingFieldNonZeroUInt64::operator=(UInt64 x) +{ + SettingFieldUInt64::operator=(x); + checkValueNonZero(); + return *this; +} + +SettingFieldNonZeroUInt64 & SettingFieldNonZeroUInt64::operator=(const DB::Field & f) +{ + SettingFieldUInt64::operator=(f); + checkValueNonZero(); + return *this; +} + +void SettingFieldNonZeroUInt64::parseFromString(const String & str) +{ + SettingFieldUInt64::parseFromString(str); + checkValueNonZero(); +} + +void SettingFieldNonZeroUInt64::checkValueNonZero() const +{ + if (value == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "A setting's value has to be greater than 0"); +} + } diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index 452f3f149ab..dc70d468851 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -627,4 +627,19 @@ struct SettingFieldCustom void readBinary(ReadBuffer & in); }; +struct SettingFieldNonZeroUInt64 : public SettingFieldUInt64 +{ +public: + explicit SettingFieldNonZeroUInt64(UInt64 x = 1); + explicit SettingFieldNonZeroUInt64(const Field & f); + + SettingFieldNonZeroUInt64 & operator=(UInt64 x); + SettingFieldNonZeroUInt64 & operator=(const Field & f); + + void parseFromString(const String & str); + +private: + void checkValueNonZero() const; +}; + } From a065231ca15684b7ebd0c1359ede037a46c6d450 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 12 Mar 2024 12:07:36 +0000 Subject: [PATCH 0108/1165] Remove unused error code --- src/Client/ConnectionPoolWithFailover.cpp | 1 - src/Client/HedgedConnectionsFactory.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 94531f58bc6..0724153b277 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -21,7 +21,6 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int ALL_CONNECTION_TRIES_FAILED; - extern const int BAD_ARGUMENTS; } diff --git a/src/Client/HedgedConnectionsFactory.cpp b/src/Client/HedgedConnectionsFactory.cpp index 6b22cc18674..0fa2bc12924 100644 --- a/src/Client/HedgedConnectionsFactory.cpp +++ b/src/Client/HedgedConnectionsFactory.cpp @@ -19,7 +19,6 @@ namespace ErrorCodes extern const int ALL_CONNECTION_TRIES_FAILED; extern const int ALL_REPLICAS_ARE_STALE; extern const int LOGICAL_ERROR; - extern const int BAD_ARGUMENTS; } HedgedConnectionsFactory::HedgedConnectionsFactory( From 3c2514bc77837b103c693890515fb1fd87e1befc Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 12 Mar 2024 18:33:32 +0000 Subject: [PATCH 0109/1165] Correct small issues --- src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp | 7 +++++-- src/Storages/System/attachSystemTables.cpp | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index ea2c71b0018..6c3a4245a96 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -66,9 +66,12 @@ protected: UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * current_end = &vec[real_block_size]; - if (step > 1) { + if (step > 1) + { iota_with_step(pos, static_cast(current_end - pos), curr, step); - } else { + } + else + { iota(pos, static_cast(current_end - pos), curr); } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 6d479bc05d7..14125cbd506 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -119,9 +119,9 @@ namespace DB void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper) { attachNoDescription(context, system_database, "one", "This table contains a single row with a single dummy UInt8 column containing the value 0. Used when the table is not specified explicitly, for example in queries like `SELECT 1`."); - attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false); - attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true); - attach(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); + attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "numbers"); + attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "numbers"); + attachNoDescription(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); attachNoDescription(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attachNoDescription(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); attach(context, system_database, "databases", "Lists all databases of the current server."); From da06ba432e7a3232688a661aac337f9994fddead Mon Sep 17 00:00:00 2001 From: unashi Date: Wed, 13 Mar 2024 12:09:35 +0800 Subject: [PATCH 0110/1165] [fix] merge and resolve problems --- src/Storages/MergeTree/MergeTreeData.cpp | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 55ab8208472..f8448be7268 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7233,27 +7233,6 @@ std::pair MergeTreeData::cloneAn scope_guard src_flushed_tmp_dir_lock; MergeTreeData::MutableDataPartPtr src_flushed_tmp_part; - /// If source part is in memory, flush it to disk and clone it already in on-disk format - /// Protect tmp dir from removing by cleanup thread with src_flushed_tmp_dir_lock - /// Construct src_flushed_tmp_part in order to delete part with its directory at destructor - if (auto src_part_in_memory = asInMemoryPart(src_part)) - { - auto flushed_part_path = *src_part_in_memory->getRelativePathForPrefix(tmp_part_prefix); - - auto tmp_src_part_file_name = fs::path(tmp_dst_part_name).filename(); - src_flushed_tmp_dir_lock = src_part->storage.getTemporaryPartDirectoryHolder(tmp_src_part_file_name); - - auto flushed_part_storage = src_part_in_memory->flushToDisk(flushed_part_path, metadata_snapshot); - - src_flushed_tmp_part = MergeTreeDataPartBuilder(*this, src_part->name, flushed_part_storage) - .withPartInfo(src_part->info) - .withPartFormatFromDisk() - .build(); - - src_flushed_tmp_part->is_temp = true; - src_part_storage = flushed_part_storage; - } - String with_copy; if (params.copy_instead_of_hardlink) with_copy = " (copying data)"; From cea5d460f8a14561d733fef69cff1721567c3fb2 Mon Sep 17 00:00:00 2001 From: unashi Date: Wed, 13 Mar 2024 15:06:28 +0800 Subject: [PATCH 0111/1165] [debug] replace 2 submodule to new version(whose version can't be found in test) --- contrib/idxd-config | 2 +- contrib/re2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/idxd-config b/contrib/idxd-config index a836ce0e420..3feff87d1f4 160000 --- a/contrib/idxd-config +++ b/contrib/idxd-config @@ -1 +1 @@ -Subproject commit a836ce0e42052a69bffbbc14239ab4097f3b77f1 +Subproject commit 3feff87d1f4add861b356f2126364750c7d00727 diff --git a/contrib/re2 b/contrib/re2 index a807e8a3aac..108914d28a7 160000 --- a/contrib/re2 +++ b/contrib/re2 @@ -1 +1 @@ -Subproject commit a807e8a3aac2cc33c77b7071efea54fcabe38e0c +Subproject commit 108914d28a79243d4300e7e651cd0a0d5883ca0f From 28bebe40a8bc457a6c743e11727269ad1edc6b0d Mon Sep 17 00:00:00 2001 From: unashi Date: Wed, 13 Mar 2024 15:17:35 +0800 Subject: [PATCH 0112/1165] [debug] change the submodule back to version which is similar to master --- contrib/idxd-config | 2 +- contrib/re2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/idxd-config b/contrib/idxd-config index 3feff87d1f4..a836ce0e420 160000 --- a/contrib/idxd-config +++ b/contrib/idxd-config @@ -1 +1 @@ -Subproject commit 3feff87d1f4add861b356f2126364750c7d00727 +Subproject commit a836ce0e42052a69bffbbc14239ab4097f3b77f1 diff --git a/contrib/re2 b/contrib/re2 index 108914d28a7..a807e8a3aac 160000 --- a/contrib/re2 +++ b/contrib/re2 @@ -1 +1 @@ -Subproject commit 108914d28a79243d4300e7e651cd0a0d5883ca0f +Subproject commit a807e8a3aac2cc33c77b7071efea54fcabe38e0c From d17212616c46f5e18c0719bedc84b1bd91e05d41 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 13 Mar 2024 17:26:45 +0800 Subject: [PATCH 0113/1165] add perf tests --- tests/performance/function_tokens.xml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/performance/function_tokens.xml diff --git a/tests/performance/function_tokens.xml b/tests/performance/function_tokens.xml new file mode 100644 index 00000000000..63b72f83df3 --- /dev/null +++ b/tests/performance/function_tokens.xml @@ -0,0 +1,3 @@ + + with 'Many years later as he faced the firing squad, Colonel Aureliano Buendia was to remember that distant afternoon when his father took him to discover ice.' as s select splitByChar(' ', materialize(s)) as w from numbers(1000000) + From c8f6d6ada9fad7145d92f49774102352d7589004 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 13 Mar 2024 10:53:57 +0100 Subject: [PATCH 0114/1165] Add missing finalize --- src/Interpreters/Cache/EvictionCandidates.cpp | 3 +++ src/Interpreters/Cache/SLRUFileCachePriority.cpp | 1 + 2 files changed, 4 insertions(+) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index fb0b3809f80..1a93c51572e 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -70,11 +70,14 @@ void EvictionCandidates::finalize(FileCacheQueryLimit::QueryContext * query_cont { for (const auto & it : invalidated_queue_entries) { + /// Remove entry from per query priority queue. if (query_context) { const auto & entry = it->getEntry(); query_context->remove(entry->key, entry->offset, lock); } + + /// Remove entry from main priority queue. it->remove(lock); } diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 543d6a03669..84133bc5c56 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -218,6 +218,7 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach } /// Make space for "downgrade" candidates. eviction_candidates.evict(); + eviction_candidates.finalize(nullptr, lock); } /// All checks passed, now we can move downgrade candidates to From dc9548d31b98d6c371d6b84c70bf98ad850bf475 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 13 Mar 2024 18:57:33 +0800 Subject: [PATCH 0115/1165] add perf test --- tests/performance/array_join.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/performance/array_join.xml b/tests/performance/array_join.xml index ca280ce28ad..b3fc24d3166 100644 --- a/tests/performance/array_join.xml +++ b/tests/performance/array_join.xml @@ -8,4 +8,6 @@ SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + + with 'clickhouse' as str select arrayJoin(range(number % 10)), materialize(str) from numbers(10000000) From cf639bab44807e6403f9392c21628a1f86558ef3 Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 13 Mar 2024 13:37:26 +0000 Subject: [PATCH 0116/1165] Fix attach bug --- src/Storages/System/attachSystemTables.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 14125cbd506..3fa55fe28c5 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -119,8 +119,8 @@ namespace DB void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper) { attachNoDescription(context, system_database, "one", "This table contains a single row with a single dummy UInt8 column containing the value 0. Used when the table is not specified explicitly, for example in queries like `SELECT 1`."); - attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "numbers"); - attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "numbers"); + attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "number"); + attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "number"); attachNoDescription(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); attachNoDescription(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attachNoDescription(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); From 16c8ee509df2f3811198086a119ed786e130a03b Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 13 Mar 2024 15:52:30 +0100 Subject: [PATCH 0117/1165] Fix tests --- src/Interpreters/Cache/EvictionCandidates.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 1a93c51572e..ddd058833b9 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -59,7 +59,7 @@ void EvictionCandidates::evict() ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedBytes, segment->range().size()); locked_key->removeFileSegment(segment->offset(), segment->lock()); - queue_it->invalidate(); + chassert(queue_it->getEntry()->size == 0); to_evict.pop_back(); } From 97b3474332b471dd795b2a32cec30da4d52cc1c2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 13 Mar 2024 15:54:05 +0100 Subject: [PATCH 0118/1165] Add assertion --- src/Interpreters/Cache/LRUFileCachePriority.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 05bbc26e602..644d8a43cb3 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -374,6 +374,7 @@ void LRUFileCachePriority::LRUIterator::invalidate() "Invalidating entry in LRU queue. Key: {}, offset: {}, previous size: {}", entry->key, entry->offset, entry->size); + chassert(entry->size != 0); cache_priority->updateSize(-entry->size); cache_priority->updateElementsCount(-1); entry->size = 0; From 83019102f3bac7bccfdd25fbbc264bcc1468dc00 Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 13 Mar 2024 17:43:48 +0000 Subject: [PATCH 0119/1165] Correct fast test ande fix perfomance issue --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 25 ++++++++++++++----- .../02992_all_columns_should_have_comment.sql | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 6c3a4245a96..52b5eeb2ca3 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -66,13 +66,13 @@ protected: UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * current_end = &vec[real_block_size]; - if (step > 1) + if (step == 1) { - iota_with_step(pos, static_cast(current_end - pos), curr, step); + iota(pos, static_cast(current_end - pos), curr); } else { - iota(pos, static_cast(current_end - pos), curr); + iota_with_step(pos, static_cast(current_end - pos), curr, step); } next += chunk_step; @@ -286,7 +286,14 @@ protected: auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = (end_value_64 - start_value_64) / this->step; - iota_with_step(pos, static_cast(size), start_value_64, step); + if (step == 1) + { + iota(pos, static_cast(size), start_value_64); + } + else + { + iota_with_step(pos, static_cast(size), start_value_64, step); + } pos += size; } }; @@ -295,9 +302,15 @@ protected: { UInt64 start_value = range.left + cursor.offset_in_range * step; /// end_value will never overflow - iota_with_step(pos, static_cast(need), start_value, step); + if (step == 1) + { + iota(pos, static_cast(need), start_value); + } + else + { + iota_with_step(pos, static_cast(need), start_value, step); + } pos += need; - provided += need; cursor.offset_in_range += need; } diff --git a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql index b984c12e285..b90faeac359 100644 --- a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql +++ b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql @@ -1,4 +1,4 @@ SYSTEM FLUSH LOGS; SELECT 'Column ' || name || ' from table ' || concat(database, '.', table) || ' should have a comment' FROM system.columns -WHERE (database = 'system') AND (comment = '') AND (table NOT ILIKE '%_log_%') AND (table NOT IN ('numbers', 'numbers_mt', 'one')) AND (default_kind != 'ALIAS'); +WHERE (database = 'system') AND (comment = '') AND (table NOT ILIKE '%_log_%') AND (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series')) AND (default_kind != 'ALIAS'); From 32410a68c136570cc19f0115a6b752f1d4cf93aa Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 13 Mar 2024 18:00:57 +0000 Subject: [PATCH 0120/1165] Fix tests --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index e28d8366aa7..22bbfc04401 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -800,7 +800,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( != parallel_replicas_before_analysis) { context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); - context->setSetting("max_parallel_replicas", UInt64{0}); + context->setSetting("max_parallel_replicas", UInt64{1}); need_analyze_again = true; } From d2b8afb98b39067d057ce0159d24d5879d284e44 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Thu, 14 Mar 2024 18:26:13 +0800 Subject: [PATCH 0121/1165] add benchmarks for column::insertmanyfrom --- src/Columns/CMakeLists.txt | 4 + src/Columns/benchmarks/CMakeLists.txt | 4 + .../benchmark_column_insert_many_from.cpp | 102 ++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 src/Columns/benchmarks/CMakeLists.txt create mode 100644 src/Columns/benchmarks/benchmark_column_insert_many_from.cpp diff --git a/src/Columns/CMakeLists.txt b/src/Columns/CMakeLists.txt index f676f415eea..1febe4f71d7 100644 --- a/src/Columns/CMakeLists.txt +++ b/src/Columns/CMakeLists.txt @@ -1,3 +1,7 @@ if (ENABLE_EXAMPLES) add_subdirectory (examples) endif () + +if (ENABLE_BENCHMARKS) + add_subdirectory(benchmarks) +endif() diff --git a/src/Columns/benchmarks/CMakeLists.txt b/src/Columns/benchmarks/CMakeLists.txt new file mode 100644 index 00000000000..47f5dfe4c59 --- /dev/null +++ b/src/Columns/benchmarks/CMakeLists.txt @@ -0,0 +1,4 @@ +clickhouse_add_executable(column_insert_many_from benchmark_column_insert_many_from.cpp) +target_link_libraries (column_insert_many_from PRIVATE + ch_contrib::gbenchmark_all + dbms) \ No newline at end of file diff --git a/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp b/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp new file mode 100644 index 00000000000..325cf5559cd --- /dev/null +++ b/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +static constexpr size_t ROWS = 65536; + +static ColumnPtr mockColumn(const DataTypePtr & type, size_t rows) +{ + const auto * type_array = typeid_cast(type.get()); + if (type_array) + { + auto data_col = mockColumn(type_array->getNestedType(), rows); + auto offset_col = ColumnArray::ColumnOffsets::create(rows); + auto & offsets = offset_col->getData(); + for (size_t i = 0; i < data_col->size(); ++i) + offsets[i] = offsets[i - 1] + (rand() % 10); + auto new_data_col = data_col->replicate(offsets); + + return ColumnArray::create(new_data_col, std::move(offset_col)); + } + + auto type_not_nullable = removeNullable(type); + auto column = type->createColumn(); + for (size_t i = 0; i < rows; ++i) + { + if (i % 100) + column->insertDefault(); + else if (isInt(type_not_nullable)) + column->insert(i); + else if (isFloat(type_not_nullable)) + { + double d = i * 1.0; + column->insert(d); + } + else if (isString(type_not_nullable)) + { + String s = "helloworld"; + column->insert(s); + } + else + column->insertDefault(); + } + return std::move(column); +} + + +static NO_INLINE void insertManyFrom(IColumn & dst, const IColumn & src) +{ + size_t size = src.size(); + dst.insertManyFrom(src, size / 2, size); +} + + +template +static void BM_insertManyFrom(benchmark::State & state) +{ + auto type = DataTypeFactory::instance().get(str_type); + auto src = mockColumn(type, ROWS); + + for (auto _ : state) + { + state.PauseTiming(); + auto dst = type->createColumn(); + dst->reserve(ROWS); + state.ResumeTiming(); + + insertManyFrom(*dst, *src); + benchmark::DoNotOptimize(dst); + } +} + +static const String type_int64 = "Int64"; +static const String type_nullable_int64 = "Nullable(Int64)"; +static const String type_string = "String"; +static const String type_nullable_string = "Nullable(String)"; +static const String type_decimal = "Decimal128(3)"; +static const String type_nullable_decimal = "Nullable(Decimal128(3))"; + +static const String type_array_int64 = "Array(Int64)"; +static const String type_array_nullable_int64 = "Array(Nullable(Int64))"; +static const String type_array_string = "Array(String)"; +static const String type_array_nullable_string = "Array(Nullable(String))"; + +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_int64); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_nullable_int64); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_string); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_nullable_string); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_decimal); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_nullable_decimal); + +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_array_int64); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_array_nullable_int64); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_array_string); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_array_nullable_string); From 64712601a4f966d98e2b3120e4a2e19f2fb428af Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 14 Mar 2024 12:03:24 +0100 Subject: [PATCH 0122/1165] Add assertions --- src/Interpreters/Cache/LRUFileCachePriority.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 644d8a43cb3..0f164f25293 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -113,6 +113,9 @@ LRUFileCachePriority::LRUQueue::iterator LRUFileCachePriority::remove(LRUQueue:: void LRUFileCachePriority::updateSize(int64_t size) { + chassert(size != 0); + chassert(size > 0 || state->current_size >= size_t(-size)); + state->current_size += size; CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size); } @@ -390,6 +393,7 @@ void LRUFileCachePriority::LRUIterator::incrementSize(size_t size, const CacheGu "Increment size with {} in LRU queue for key: {}, offset: {}, previous size: {}", size, entry->key, entry->offset, entry->size); + chassert(size); cache_priority->updateSize(size); entry->size += size; } @@ -404,6 +408,9 @@ void LRUFileCachePriority::LRUIterator::decrementSize(size_t size) "Decrement size with {} in LRU queue for key: {}, offset: {}, previous size: {}", size, entry->key, entry->offset, entry->size); + chassert(size); + chassert(entry->size >= size); + cache_priority->updateSize(-size); entry->size -= size; } From f63e1b48e63443aa63acd53962b99c7d645a9507 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 14 Mar 2024 11:21:49 +0000 Subject: [PATCH 0123/1165] Temporarily increase build limit with sanitizers --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ffb4789dc9..7a07cf2b436 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,7 @@ if (ENABLE_CHECK_HEAVY_BUILDS) # -fsanitize=memory and address are too heavy if (SANITIZE) - set (RLIMIT_DATA 10000000000) # 10G + set (RLIMIT_DATA 15000000000) # 10G endif() set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) From a0044f3aa4d5c297229be6f6829f8d7b5e7d75c5 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 14 Mar 2024 11:44:35 +0000 Subject: [PATCH 0124/1165] Fix ATTACH query with external ON CLUSTER --- src/Interpreters/InterpreterCreateQuery.cpp | 7 +++++-- tests/integration/test_external_cluster/test.py | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index edd7452c130..fb421ee0a5e 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1087,8 +1087,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) // If this is a stub ATTACH query, read the query definition from the database if (create.attach && !create.storage && !create.columns_list) { - auto database = DatabaseCatalog::instance().getDatabase(database_name); - if (database->shouldReplicateQuery(getContext(), query_ptr)) + auto database = DatabaseCatalog::instance().tryGetDatabase(database_name); + if (database && database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); create.setDatabase(database_name); @@ -1099,6 +1099,9 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (!create.cluster.empty()) return executeQueryOnCluster(create); + if (!database) + throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(database_name)); + /// For short syntax of ATTACH query we have to lock table name here, before reading metadata /// and hold it until table is attached if (likely(need_ddl_guard)) diff --git a/tests/integration/test_external_cluster/test.py b/tests/integration/test_external_cluster/test.py index 2ed8ada3df4..6fa9cd16dbc 100644 --- a/tests/integration/test_external_cluster/test.py +++ b/tests/integration/test_external_cluster/test.py @@ -46,6 +46,12 @@ def test_ddl(started_cluster): control_node.query( "ALTER TABLE test_db.test_table ON CLUSTER 'external' add column data String" ) + control_node.query("DETACH TABLE test_db.test_table ON CLUSTER 'external'") + + expected = "" + assert_create_query(data_node, "test_db", "test_table", expected) + + control_node.query("ATTACH TABLE test_db.test_table ON CLUSTER 'external'") expected = "CREATE TABLE test_db.test_table (`id` Int64, `data` String) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192" assert_create_query(data_node, "test_db", "test_table", expected) From bb5e8f52a3fad759007ea6b7684f770f9646c103 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 14 Mar 2024 14:27:41 +0000 Subject: [PATCH 0125/1165] Kek --- CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a07cf2b436..4799e055698 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,19 +54,19 @@ endif () # ccache ignore it. option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF) if (ENABLE_CHECK_HEAVY_BUILDS) - # set DATA (since RSS does not work since 2.6.x+) to 5G - set (RLIMIT_DATA 5000000000) - # set VIRT (RLIMIT_AS) to 10G (DATA*10) - set (RLIMIT_AS 10000000000) - # set CPU time limit to 1000 seconds - set (RLIMIT_CPU 1000) + # # set DATA (since RSS does not work since 2.6.x+) to 5G + # set (RLIMIT_DATA 5000000000) + # # set VIRT (RLIMIT_AS) to 10G (DATA*10) + # set (RLIMIT_AS 10000000000) + # # set CPU time limit to 1000 seconds + # set (RLIMIT_CPU 1000) - # -fsanitize=memory and address are too heavy - if (SANITIZE) - set (RLIMIT_DATA 15000000000) # 10G - endif() + # # -fsanitize=memory and address are too heavy + # if (SANITIZE) + # set (RLIMIT_DATA 15000000000) # 10G + # endif() - set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) + # set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) endif () if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") From 75aed5ce8651fa1aff93bb726418de3df413ead6 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 14 Mar 2024 16:26:38 +0000 Subject: [PATCH 0126/1165] fix consecutive keys optimization for nullable keys --- src/Common/ColumnsHashingImpl.h | 88 +++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/src/Common/ColumnsHashingImpl.h b/src/Common/ColumnsHashingImpl.h index 7116160e94c..c8d62fa7e6b 100644 --- a/src/Common/ColumnsHashingImpl.h +++ b/src/Common/ColumnsHashingImpl.h @@ -46,29 +46,45 @@ struct LastElementCacheStats namespace columns_hashing_impl { -template -struct LastElementCache +struct LastElementCacheBase { - static constexpr bool consecutive_keys_optimization = consecutive_keys_optimization_; - - Value value; bool empty = true; bool found = false; UInt64 misses = 0; - bool check(const Value & value_) const { return value == value_; } + void onNewValue(bool is_found) + { + empty = false; + found = is_found; + ++misses; + } + + bool hasOnlyOneValue() const { return found && misses == 1; } +}; + +template struct LastElementCache; + +template +struct LastElementCache : public LastElementCacheBase +{ + Value value; + bool is_null = false; + + template + bool check(const Key & key) const { return !is_null && value.first == key; } + + bool check(const Value & rhs) const { return !is_null && value == rhs; } +}; + +template +struct LastElementCache : public LastElementCacheBase +{ + Value value; template bool check(const Key & key) const { return value.first == key; } - bool hasOnlyOneValue() const { return found && misses == 1; } - UInt64 getMisses() const { return misses; } -}; - -template -struct LastElementCache -{ - static constexpr bool consecutive_keys_optimization = false; + bool check(const Value & rhs) const { return value == rhs; } }; template @@ -162,7 +178,7 @@ public: using EmplaceResult = EmplaceResultImpl; using FindResult = FindResultImpl; static constexpr bool has_mapped = !std::is_same_v; - using Cache = LastElementCache; + using Cache = LastElementCache; static HashMethodContextPtr createContext(const HashMethodContext::Settings &) { return nullptr; } @@ -173,6 +189,15 @@ public: { if (isNullAt(row)) { + if constexpr (consecutive_keys_optimization) + { + if (!cache.is_null) + { + cache.onNewValue(true); + cache.is_null = true; + } + } + bool has_null_key = data.hasNullKeyData(); data.hasNullKeyData() = true; @@ -194,10 +219,21 @@ public: { if (isNullAt(row)) { + bool has_null_key = data.hasNullKeyData(); + + if constexpr (consecutive_keys_optimization) + { + if (!cache.is_null) + { + cache.onNewValue(has_null_key); + cache.is_null = true; + } + } + if constexpr (has_mapped) - return FindResult(&data.getNullKeyData(), data.hasNullKeyData(), 0); + return FindResult(&data.getNullKeyData(), has_null_key, 0); else - return FindResult(data.hasNullKeyData(), 0); + return FindResult(has_null_key, 0); } } @@ -232,7 +268,7 @@ public: ALWAYS_INLINE UInt64 getCacheMissesSinceLastReset() const { if constexpr (consecutive_keys_optimization) - return cache.getMisses(); + return cache.misses; return 0; } @@ -304,9 +340,10 @@ protected: if constexpr (consecutive_keys_optimization) { - cache.found = true; - cache.empty = false; - ++cache.misses; + cache.onNewValue(true); + + if constexpr (nullable) + cache.is_null = false; if constexpr (has_mapped) { @@ -347,17 +384,16 @@ protected: if constexpr (consecutive_keys_optimization) { - cache.found = it != nullptr; - cache.empty = false; - ++cache.misses; + cache.onNewValue(it != nullptr); + + if constexpr (nullable) + cache.is_null = false; if constexpr (has_mapped) { cache.value.first = key; if (it) - { cache.value.second = it->getMapped(); - } } else { From e56d472602d066612abb806d76307c2626ed9ff2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 14 Mar 2024 17:43:14 +0100 Subject: [PATCH 0127/1165] Fix correctness --- src/Interpreters/Cache/EvictionCandidates.cpp | 34 +++++++++++++------ src/Interpreters/Cache/EvictionCandidates.h | 2 +- src/Interpreters/Cache/FileCache.cpp | 15 ++++++-- src/Interpreters/Cache/Metadata.cpp | 14 ++++---- src/Interpreters/Cache/Metadata.h | 18 ++++++++-- 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index ddd058833b9..54917f2fd99 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -14,6 +14,13 @@ namespace DB EvictionCandidates::~EvictionCandidates() { + for (const auto & iterator : queue_entries_to_invalidate) + { + /// If there was an exception between evict and finalize phase + /// for some eviction candidate, we need to reset its entry now. + iterator->invalidate(); + } + for (const auto & [key, key_candidates] : candidates) { for (const auto & candidate : key_candidates.candidates) @@ -38,6 +45,7 @@ void EvictionCandidates::evict() return; auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::FilesystemCacheEvictMicroseconds); + queue_entries_to_invalidate.reserve(candidates_size); for (auto & [key, key_candidates] : candidates) { @@ -45,40 +53,44 @@ void EvictionCandidates::evict() if (!locked_key) continue; /// key could become invalid after we released the key lock above, just skip it. - auto & to_evict = key_candidates.candidates; - while (!to_evict.empty()) + while (!key_candidates.candidates.empty()) { - auto & candidate = to_evict.back(); + auto & candidate = key_candidates.candidates.back(); chassert(candidate->releasable()); const auto segment = candidate->file_segment; - auto queue_it = segment->getQueueIterator(); - chassert(queue_it); + auto iterator = segment->getQueueIterator(); + chassert(iterator); ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedFileSegments); ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedBytes, segment->range().size()); - locked_key->removeFileSegment(segment->offset(), segment->lock()); - chassert(queue_it->getEntry()->size == 0); + locked_key->removeFileSegment( + segment->offset(), segment->lock(), /* can_be_broken */false, /* invalidate_queue_entry */false); - to_evict.pop_back(); + queue_entries_to_invalidate.push_back(iterator); + key_candidates.candidates.pop_back(); } } } void EvictionCandidates::finalize(FileCacheQueryLimit::QueryContext * query_context, const CacheGuard::Lock & lock) { - for (const auto & it : invalidated_queue_entries) + while (!queue_entries_to_invalidate.empty()) { + auto iterator = queue_entries_to_invalidate.back(); + iterator->invalidate(); + queue_entries_to_invalidate.pop_back(); + /// Remove entry from per query priority queue. if (query_context) { - const auto & entry = it->getEntry(); + const auto & entry = iterator->getEntry(); query_context->remove(entry->key, entry->offset, lock); } /// Remove entry from main priority queue. - it->remove(lock); + iterator->remove(lock); } if (finalize_eviction_func) diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index 7859762be09..81681dd7d00 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -33,8 +33,8 @@ private: std::unordered_map candidates; size_t candidates_size = 0; - std::vector invalidated_queue_entries; FinalizeEvictionFunc finalize_eviction_func; + std::vector queue_entries_to_invalidate; }; using EvictionCandidatesPtr = std::unique_ptr; diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 90508d74554..051068d9fa4 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -841,14 +841,23 @@ bool FileCache::tryReserve( if (!file_segment.getKeyMetadata()->createBaseDirectory()) return false; - /// Remove eviction candidates from filesystem. - eviction_candidates.evict(); + try + { + /// Remove eviction candidates from filesystem. + eviction_candidates.evict(); + } + catch (...) + { + /// Invalidate queue entries if some successed to be removed. + eviction_candidates.finalize(query_context.get(), cache_lock); + throw; + } /// Take cache lock again. if (release_lock) cache_lock.lock(); - /// Remove invalidated queue entries and execute (only for SLRU) finalize func. + /// Invalidate and remove queue entries and execute (only for SLRU) finalize func. eviction_candidates.finalize(query_context.get(), cache_lock); /// Space reservation is incremental, so file_segment_metadata is created first (with state Empty), diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index eb3b4e44c8c..d3fb699a115 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -895,32 +895,34 @@ bool LockedKey::removeAllFileSegments(bool if_releasable) return removed_all; } -KeyMetadata::iterator LockedKey::removeFileSegment(size_t offset, bool can_be_broken) +KeyMetadata::iterator LockedKey::removeFileSegment(size_t offset, bool can_be_broken, bool invalidate_queue_entry) { auto it = key_metadata->find(offset); if (it == key_metadata->end()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no offset {}", offset); auto file_segment = it->second->file_segment; - return removeFileSegmentImpl(it, file_segment->lock(), can_be_broken); + return removeFileSegmentImpl(it, file_segment->lock(), can_be_broken, invalidate_queue_entry); } KeyMetadata::iterator LockedKey::removeFileSegment( size_t offset, const FileSegmentGuard::Lock & segment_lock, - bool can_be_broken) + bool can_be_broken, + bool invalidate_queue_entry) { auto it = key_metadata->find(offset); if (it == key_metadata->end()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no offset {} in key {}", offset, getKey()); - return removeFileSegmentImpl(it, segment_lock, can_be_broken); + return removeFileSegmentImpl(it, segment_lock, can_be_broken, invalidate_queue_entry); } KeyMetadata::iterator LockedKey::removeFileSegmentImpl( KeyMetadata::iterator it, const FileSegmentGuard::Lock & segment_lock, - bool can_be_broken) + bool can_be_broken, + bool invalidate_queue_entry) { auto file_segment = it->second->file_segment; @@ -930,7 +932,7 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl( chassert(can_be_broken || file_segment->assertCorrectnessUnlocked(segment_lock)); - if (file_segment->queue_iterator) + if (file_segment->queue_iterator && invalidate_queue_entry) file_segment->queue_iterator->invalidate(); file_segment->detach(segment_lock, *this); diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index c02127cdef3..adcc1545307 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -269,8 +269,16 @@ struct LockedKey : private boost::noncopyable bool removeAllFileSegments(bool if_releasable = true); - KeyMetadata::iterator removeFileSegment(size_t offset, const FileSegmentGuard::Lock &, bool can_be_broken = false); - KeyMetadata::iterator removeFileSegment(size_t offset, bool can_be_broken = false); + KeyMetadata::iterator removeFileSegment( + size_t offset, + const FileSegmentGuard::Lock &, + bool can_be_broken = false, + bool invalidate_queue_entry = true); + + KeyMetadata::iterator removeFileSegment( + size_t offset, + bool can_be_broken = false, + bool invalidate_queue_entry = true); void shrinkFileSegmentToDownloadedSize(size_t offset, const FileSegmentGuard::Lock &); @@ -289,7 +297,11 @@ struct LockedKey : private boost::noncopyable std::string toString() const; private: - KeyMetadata::iterator removeFileSegmentImpl(KeyMetadata::iterator it, const FileSegmentGuard::Lock &, bool can_be_broken = false); + KeyMetadata::iterator removeFileSegmentImpl( + KeyMetadata::iterator it, + const FileSegmentGuard::Lock &, + bool can_be_broken = false, + bool invalidate_queue_entry = true); const std::shared_ptr key_metadata; KeyGuard::Lock lock; /// `lock` must be destructed before `key_metadata`. From e3666e42a9864623f0f509c90a81bba6581c350e Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 14 Mar 2024 16:47:37 +0000 Subject: [PATCH 0128/1165] add a test --- .../03009_consecutive_keys_nullable.reference | 16 ++++++ .../03009_consecutive_keys_nullable.sql | 56 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 tests/queries/0_stateless/03009_consecutive_keys_nullable.reference create mode 100644 tests/queries/0_stateless/03009_consecutive_keys_nullable.sql diff --git a/tests/queries/0_stateless/03009_consecutive_keys_nullable.reference b/tests/queries/0_stateless/03009_consecutive_keys_nullable.reference new file mode 100644 index 00000000000..1c8064290c6 --- /dev/null +++ b/tests/queries/0_stateless/03009_consecutive_keys_nullable.reference @@ -0,0 +1,16 @@ +\N 1 1 +1 2 0 +\N 1 1 +1 2 0 +\N 3 3 +1 3 0 +\N 1 1 +1 2 0 +\N 2 2 +1 1 0 +t_nullable_keys_1 0 +t_nullable_keys_2 0 +t_nullable_keys_3 1 +t_nullable_keys_4 1 +t_nullable_keys_5 0 +t_nullable_keys_6 0 diff --git a/tests/queries/0_stateless/03009_consecutive_keys_nullable.sql b/tests/queries/0_stateless/03009_consecutive_keys_nullable.sql new file mode 100644 index 00000000000..ee2cb5a171f --- /dev/null +++ b/tests/queries/0_stateless/03009_consecutive_keys_nullable.sql @@ -0,0 +1,56 @@ +DROP TABLE IF EXISTS t_nullable_keys_1; + +CREATE TABLE t_nullable_keys_1 (x Nullable(Int64)) ENGINE = Memory; +INSERT INTO t_nullable_keys_1 VALUES (1), (1), (NULL); +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_1 GROUP BY x; + +DROP TABLE t_nullable_keys_1; + +DROP TABLE IF EXISTS t_nullable_keys_2; + +CREATE TABLE t_nullable_keys_2 (x Nullable(Int64)) ENGINE = Memory; +INSERT INTO t_nullable_keys_2 VALUES (NULL), (1), (1); +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_2 GROUP BY x; + +DROP TABLE t_nullable_keys_2; + +DROP TABLE IF EXISTS t_nullable_keys_3; + +CREATE TABLE t_nullable_keys_3 (x Nullable(Int64)) ENGINE = Memory; +INSERT INTO t_nullable_keys_3 VALUES (NULL), (NULL), (NULL); +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_3 GROUP BY x; + +DROP TABLE t_nullable_keys_3; + +DROP TABLE IF EXISTS t_nullable_keys_4; + +CREATE TABLE t_nullable_keys_4 (x Nullable(Int64)) ENGINE = Memory; +INSERT INTO t_nullable_keys_4 VALUES (1), (1), (1); +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_4 GROUP BY x; + +DROP TABLE t_nullable_keys_4; + +DROP TABLE IF EXISTS t_nullable_keys_5; + +CREATE TABLE t_nullable_keys_5 (x Nullable(Int64)) ENGINE = Memory; +INSERT INTO t_nullable_keys_5 VALUES (1), (NULL), (1); +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_5 GROUP BY x; + +DROP TABLE t_nullable_keys_5; + +DROP TABLE IF EXISTS t_nullable_keys_6; + +CREATE TABLE t_nullable_keys_6 (x Nullable(Int64)) ENGINE = Memory; +INSERT INTO t_nullable_keys_6 VALUES (NULL), (1), (NULL); +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_6 GROUP BY x; + +DROP TABLE t_nullable_keys_6; + +SYSTEM FLUSH LOGS; + +SELECT + splitByChar('.', tables[1])[2] AS table, + ProfileEvents['AggregationOptimizedEqualRangesOfKeys'] > 0 +FROM system.query_log +WHERE type = 'QueryFinish' AND current_database = currentDatabase() AND query LIKE '%SELECT%FROM%t_nullable_keys_%' +ORDER BY table; From 095d53a67cd02a924d2e43c9d906a152380eea3c Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 14 Mar 2024 17:57:36 +0100 Subject: [PATCH 0129/1165] Add explanation --- src/Interpreters/Cache/EvictionCandidates.cpp | 10 ++++++++++ src/Interpreters/Cache/FileCache.cpp | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 54917f2fd99..34be84e5407 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -65,6 +65,16 @@ void EvictionCandidates::evict() ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedFileSegments); ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedBytes, segment->range().size()); + /// We remove file segment, but do not invalidate queue entry now, + /// we will invalidate it after all eviction candidates are removed (in finalize() method), + /// because invalidation of queue entries needs to be done under cache lock. + /// Why? + /// Firstly, as long as queue entry exists, the corresponding space in cache is considered to be hold, + /// and once it is invalidated - the space is released. + /// Secondly, after evict() and finalize() stages we will also add back "reserved size" + /// (<= actually released size), but until we do this - we cannot allow other threads to think that + /// this released space is free, as it is not - it is removed in favour of some reserver + /// so we can make it visibly free only for that particular reserver. locked_key->removeFileSegment( segment->offset(), segment->lock(), /* can_be_broken */false, /* invalidate_queue_entry */false); diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 051068d9fa4..2daf5e1e1e4 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -848,7 +848,7 @@ bool FileCache::tryReserve( } catch (...) { - /// Invalidate queue entries if some successed to be removed. + /// Invalidate queue entries if some succeeded to be removed. eviction_candidates.finalize(query_context.get(), cache_lock); throw; } From cd912074a5a94dc80daf4104942e0a9d3ed26e45 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 14 Mar 2024 18:29:07 +0100 Subject: [PATCH 0130/1165] Fixup --- src/DataTypes/ObjectUtils.cpp | 14 ++++++-------- src/DataTypes/ObjectUtils.h | 4 +--- .../ClusterProxy/SelectStreamFactory.cpp | 19 ++++++++++++------- .../ClusterProxy/SelectStreamFactory.h | 7 +++++-- src/Processors/QueryPlan/ReadFromRemote.cpp | 6 +++--- .../01455_opentelemetry_distributed.reference | 4 ++-- 6 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index ccfa0a28f13..cdd95da6c00 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -959,24 +959,22 @@ void replaceMissedSubcolumnsByConstants( /// @expected_columns and @available_columns contain descriptions /// of extended Object columns. -MissingObjectList replaceMissedSubcolumnsByConstants( +bool replaceMissedSubcolumnsByConstants( const ColumnsDescription & expected_columns, const ColumnsDescription & available_columns, QueryTreeNodePtr & query, const ContextPtr & context [[maybe_unused]]) { - MissingObjectList missed_list; + bool has_missing_objects = false; NamesAndTypes missed_names_types = calculateMissedSubcolumns(expected_columns, available_columns); if (missed_names_types.empty()) - return missed_list; + return has_missing_objects; auto * query_node = query->as(); if (!query_node) - return missed_list; - - missed_list.reserve(missed_names_types.size()); + return has_missing_objects; auto table_expression = extractLeftTableExpression(query_node->getJoinTree()); @@ -987,12 +985,12 @@ MissingObjectList replaceMissedSubcolumnsByConstants( constant->setAlias(table_expression->getAlias() + "." + name); column_name_to_node[name] = buildCastFunction(constant, type, context); - missed_list.push_back({ constant->getValueStringRepresentation() + "_" + constant->getResultType()->getName(), table_expression->getAlias() + "." + name }); + has_missing_objects = true; } replaceColumns(query, table_expression, column_name_to_node); - return missed_list; + return has_missing_objects; } Field FieldVisitorReplaceScalars::operator()(const Array & x) const diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 6ef19baf5ae..7b171056f06 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -100,9 +100,7 @@ void replaceMissedSubcolumnsByConstants( const ColumnsDescription & available_columns, ASTPtr query); -using MissingObjectList = std::vector>; - -MissingObjectList replaceMissedSubcolumnsByConstants( +bool replaceMissedSubcolumnsByConstants( const ColumnsDescription & expected_columns, const ColumnsDescription & available_columns, QueryTreeNodePtr & query, diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 4fccd83c8c0..ab301e01d0a 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -130,6 +130,7 @@ void SelectStreamFactory::createForShard( createForShardImpl( shard_info, query_ast, + {}, main_table, table_func_ptr, std::move(context), @@ -143,6 +144,7 @@ void SelectStreamFactory::createForShard( void SelectStreamFactory::createForShardImpl( const Cluster::ShardInfo & shard_info, const ASTPtr & query_ast, + const QueryTreeNodePtr & query_tree, const StorageID & main_table, const ASTPtr & table_func_ptr, ContextPtr context, @@ -151,13 +153,13 @@ void SelectStreamFactory::createForShardImpl( UInt32 shard_count, bool parallel_replicas_enabled, AdditionalShardFilterGenerator shard_filter_generator, - MissingObjectList missed_list) + bool has_missing_objects) { auto emplace_local_stream = [&]() { Block shard_header; if (context->getSettingsRef().allow_experimental_analyzer) - shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_ast, context, SelectQueryOptions(processed_stage).analyze()); + shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_tree, context, SelectQueryOptions(processed_stage).analyze()); else shard_header = header; @@ -169,15 +171,16 @@ void SelectStreamFactory::createForShardImpl( { Block shard_header; if (context->getSettingsRef().allow_experimental_analyzer) - shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_ast, context, SelectQueryOptions(processed_stage).analyze()); + shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_tree, context, SelectQueryOptions(processed_stage).analyze()); else shard_header = header; remote_shards.emplace_back(Shard{ .query = query_ast, + .query_tree = query_tree, .main_table = main_table, .header = shard_header, - .missing_object_list = std::move(missed_list), + .has_missing_objects = has_missing_objects, .shard_info = shard_info, .lazy = lazy, .local_delay = local_delay, @@ -300,15 +303,17 @@ void SelectStreamFactory::createForShard( auto it = objects_by_shard.find(shard_info.shard_num); QueryTreeNodePtr modified_query = query_tree; - MissingObjectList missed_list; + + bool has_missing_objects = false; if (it != objects_by_shard.end()) - missed_list = replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); + has_missing_objects = replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); auto query_ast = queryNodeToDistributedSelectQuery(modified_query); createForShardImpl( shard_info, query_ast, + modified_query, main_table, table_func_ptr, std::move(context), @@ -317,7 +322,7 @@ void SelectStreamFactory::createForShard( shard_count, parallel_replicas_enabled, std::move(shard_filter_generator), - std::move(missed_list)); + has_missing_objects); } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 61694830b3d..760281284fd 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -54,11 +54,13 @@ public: { /// Query and header may be changed depending on shard. ASTPtr query; + QueryTreeNodePtr query_tree; + /// Used to check the table existence on remote node StorageID main_table; Block header; - MissingObjectList missing_object_list; + bool has_missing_objects = false; Cluster::ShardInfo shard_info; @@ -110,6 +112,7 @@ private: void createForShardImpl( const Cluster::ShardInfo & shard_info, const ASTPtr & query_ast, + const QueryTreeNodePtr & query_tree, const StorageID & main_table, const ASTPtr & table_func_ptr, ContextPtr context, @@ -118,7 +121,7 @@ private: UInt32 shard_count, bool parallel_replicas_enabled, AdditionalShardFilterGenerator shard_filter_generator, - MissingObjectList missed_list = {}); + bool has_missing_objects = false); }; } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 72848a37f6e..8c455883ab2 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -218,7 +218,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); - addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); + addConvertingActions(pipes.back(), output_stream->header, shard.has_missing_objects); } void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard) @@ -299,7 +299,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); + addConvertingActions(pipes.back(), output_stream->header, shard.has_missing_objects); } } else @@ -328,7 +328,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); + addConvertingActions(pipes.back(), output_stream->header, shard.has_missing_objects); } } diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index 2920b387aa2..b04a3a5ea82 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -1,9 +1,9 @@ ===http=== {"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} -{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1` SETTINGS allow_experimental_analyzer = 1","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} -{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1` SETTINGS allow_experimental_analyzer = 1","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"total spans":"3","unique spans":"3","unique non-zero parent spans":"3"} {"initial query spans with proper parent":"2"} From 5020741b253f66967ef84707c98a2fd201d5080d Mon Sep 17 00:00:00 2001 From: Austin Kothig Date: Thu, 14 Mar 2024 14:34:45 -0700 Subject: [PATCH 0131/1165] Add support for nanosecond level precision. --- .../functions/date-time-functions.md | 4 +- .../functions/date-time-functions.md | 4 +- .../functions/date-time-functions.md | 1 + src/Functions/DateTimeTransforms.h | 38 +++++++++------- src/Functions/dateDiff.cpp | 43 +++++++++++++------ .../00538_datediff_plural_units.reference | 1 + .../00538_datediff_plural_units.sql | 1 + .../0_stateless/02814_age_datediff.reference | 6 ++- .../0_stateless/02814_age_datediff.sql | 3 ++ 9 files changed, 69 insertions(+), 32 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index ba7695af3fa..46ec9f01152 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1670,7 +1670,7 @@ Like [fromDaysSinceYearZero](#fromDaysSinceYearZero) but returns a [Date32](../. ## age -Returns the `unit` component of the difference between `startdate` and `enddate`. The difference is calculated using a precision of 1 microsecond. +Returns the `unit` component of the difference between `startdate` and `enddate`. The difference is calculated using a precision of 1 nanosecond. E.g. the difference between `2021-12-29` and `2022-01-01` is 3 days for `day` unit, 0 months for `month` unit, 0 years for `year` unit. For an alternative to `age`, see function `date\_diff`. @@ -1686,6 +1686,7 @@ age('unit', startdate, enddate, [timezone]) - `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md). Possible values: + - `nanosecond` `nanoseconds` `ns` - `microsecond` `microseconds` `us` `u` - `millisecond` `milliseconds` `ms` - `second` `seconds` `ss` `s` @@ -1763,6 +1764,7 @@ Aliases: `dateDiff`, `DATE_DIFF`, `timestampDiff`, `timestamp_diff`, `TIMESTAMP_ - `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md). Possible values: + - `nanosecond` `nanoseconds` `ns` - `microsecond` `microseconds` `us` `u` - `millisecond` `milliseconds` `ms` - `second` `seconds` `ss` `s` diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index cbbb456aa80..65c482e9914 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -627,7 +627,7 @@ SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(d ## age -Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 микросекунду. +Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 наносекунда. Например, разница между `2021-12-29` и `2022-01-01` 3 дня для единицы `day`, 0 месяцев для единицы `month`, 0 лет для единицы `year`. **Синтаксис** @@ -641,6 +641,7 @@ age('unit', startdate, enddate, [timezone]) - `unit` — единица измерения времени, в которой будет выражено возвращаемое значение функции. [String](../../sql-reference/data-types/string.md). Возможные значения: + - `nanosecond` (возможные сокращения: `ns`) - `microsecond` (возможные сокращения: `us`, `u`) - `millisecond` (возможные сокращения: `ms`) - `second` (возможные сокращения: `ss`, `s`) @@ -716,6 +717,7 @@ date_diff('unit', startdate, enddate, [timezone]) - `unit` — единица измерения времени, в которой будет выражено возвращаемое значение функции. [String](../../sql-reference/data-types/string.md). Возможные значения: + - `nanosecond` (возможные сокращения: `ns`) - `microsecond` (возможные сокращения: `us`, `u`) - `millisecond` (возможные сокращения: `ms`) - `second` (возможные сокращения: `ss`, `s`) diff --git a/docs/zh/sql-reference/functions/date-time-functions.md b/docs/zh/sql-reference/functions/date-time-functions.md index e4b70322477..d6493ffe605 100644 --- a/docs/zh/sql-reference/functions/date-time-functions.md +++ b/docs/zh/sql-reference/functions/date-time-functions.md @@ -643,6 +643,7 @@ date_diff('unit', startdate, enddate, [timezone]) - `unit` — `value`对应的时间单位。类型为[String](../../sql-reference/data-types/string.md)。 可能的值： + - `nanosecond` - `microsecond` - `millisecond` - `second` diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 70b2a7a83b4..a92680ac124 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -21,6 +21,7 @@ namespace DB { +static constexpr auto nanosecond_multiplier = 1000000000; static constexpr auto microsecond_multiplier = 1000000; static constexpr auto millisecond_multiplier = 1000; @@ -1902,9 +1903,10 @@ struct ToRelativeSubsecondNumImpl { static constexpr auto name = "toRelativeSubsecondNumImpl"; - static Int64 execute(const DateTime64 & t, DateTime64::NativeType scale, const DateLUTImpl &) + static Int64 execute(const DateTime64 & t, const DateTime64::NativeType scale, const DateLUTImpl &) { - static_assert(scale_multiplier == 1000 || scale_multiplier == 1000000); + static_assert( + scale_multiplier == millisecond_multiplier || scale_multiplier == microsecond_multiplier || scale_multiplier == nanosecond_multiplier); if (scale == scale_multiplier) return t.value; if (scale > scale_multiplier) @@ -2030,13 +2032,14 @@ struct DateTimeComponentsWithFractionalPart : public DateLUTImpl::DateTimeCompon { UInt16 millisecond; UInt16 microsecond; + UInt16 nanosecond; }; struct ToDateTimeComponentsImpl { static constexpr auto name = "toDateTimeComponents"; - static DateTimeComponentsWithFractionalPart execute(const DateTime64 & t, DateTime64::NativeType scale_multiplier, const DateLUTImpl & time_zone) + static DateTimeComponentsWithFractionalPart execute(const DateTime64 & t, const DateTime64::NativeType scale_multiplier, const DateLUTImpl & time_zone) { auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); @@ -2045,28 +2048,33 @@ struct ToDateTimeComponentsImpl components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional; --components.whole; } - Int64 fractional = components.fractional; - if (scale_multiplier > microsecond_multiplier) - fractional = fractional / (scale_multiplier / microsecond_multiplier); - else if (scale_multiplier < microsecond_multiplier) - fractional = fractional * (microsecond_multiplier / scale_multiplier); - constexpr Int64 divider = microsecond_multiplier/ millisecond_multiplier; - UInt16 millisecond = static_cast(fractional / divider); - UInt16 microsecond = static_cast(fractional % divider); - return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(components.whole), millisecond, microsecond}; + // Normalize the dividers between microseconds and nanoseconds w.r.t. the scale. + Int64 microsecond_divider = (millisecond_multiplier * scale_multiplier) / microsecond_multiplier; + Int64 nanosecond_divider = scale_multiplier / microsecond_multiplier; + + // Protect against division by zero for smaller scale multipliers. + microsecond_divider = (microsecond_divider ? microsecond_divider : 1); + nanosecond_divider = (nanosecond_divider ? nanosecond_divider : 1); + + const Int64 & fractional = components.fractional; + UInt16 millisecond = static_cast(fractional / microsecond_divider); + UInt16 microsecond = static_cast((fractional % microsecond_divider) / nanosecond_divider); + UInt16 nanosecond = static_cast(fractional % nanosecond_divider); + + return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(components.whole), millisecond, microsecond, nanosecond}; } static DateTimeComponentsWithFractionalPart execute(UInt32 t, const DateLUTImpl & time_zone) { - return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(static_cast(t)), 0, 0}; + return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(static_cast(t)), 0, 0, 0}; } static DateTimeComponentsWithFractionalPart execute(Int32 d, const DateLUTImpl & time_zone) { - return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(ExtendedDayNum(d)), 0, 0}; + return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(ExtendedDayNum(d)), 0, 0, 0}; } static DateTimeComponentsWithFractionalPart execute(UInt16 d, const DateLUTImpl & time_zone) { - return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(DayNum(d)), 0, 0}; + return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(DayNum(d)), 0, 0, 0}; } using FactorTransform = ZeroTransform; diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index f75e6eb4fc8..85b569fb634 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -177,10 +177,10 @@ public: DateTimeComponentsWithFractionalPart a_comp; DateTimeComponentsWithFractionalPart b_comp; Int64 adjust_value; - auto x_microseconds = TransformDateTime64>(transform_x.getScaleMultiplier()).execute(x, timezone_x); - auto y_microseconds = TransformDateTime64>(transform_y.getScaleMultiplier()).execute(y, timezone_y); + auto x_nanoseconds = TransformDateTime64>(transform_x.getScaleMultiplier()).execute(x, timezone_x); + auto y_nanoseconds = TransformDateTime64>(transform_y.getScaleMultiplier()).execute(y, timezone_y); - if (x_microseconds <= y_microseconds) + if (x_nanoseconds <= y_nanoseconds) { a_comp = TransformDateTime64(transform_x.getScaleMultiplier()).execute(x, timezone_x); b_comp = TransformDateTime64(transform_y.getScaleMultiplier()).execute(y, timezone_y); @@ -193,7 +193,6 @@ public: adjust_value = 1; } - if constexpr (std::is_same_v>>) { if ((a_comp.date.month > b_comp.date.month) @@ -202,7 +201,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -215,7 +215,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -225,7 +226,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -237,7 +239,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -246,7 +249,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -254,25 +258,34 @@ public: if ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) { if ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) { if ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))) res += adjust_value; } - else if constexpr (std::is_same_v>>) + else if constexpr (std::is_same_v>>) { - if (a_comp.microsecond > b_comp.microsecond) + if ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))) + res += adjust_value; + } + else if constexpr (std::is_same_v>>) + { + if (a_comp.nanosecond > b_comp.nanosecond) res += adjust_value; } return res; @@ -401,6 +414,8 @@ public: impl.template dispatchForColumns>(x, y, timezone_x, timezone_y, res->getData()); else if (unit == "microsecond" || unit == "microseconds" || unit == "us" || unit == "u") impl.template dispatchForColumns>(x, y, timezone_x, timezone_y, res->getData()); + else if (unit == "nanosecond" || unit == "nanoseconds" || unit == "ns") + impl.template dispatchForColumns>(x, y, timezone_x, timezone_y, res->getData()); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function {} does not support '{}' unit", getName(), unit); diff --git a/tests/queries/0_stateless/00538_datediff_plural_units.reference b/tests/queries/0_stateless/00538_datediff_plural_units.reference index ebe63974df8..885f8a1cfe2 100644 --- a/tests/queries/0_stateless/00538_datediff_plural_units.reference +++ b/tests/queries/0_stateless/00538_datediff_plural_units.reference @@ -8,3 +8,4 @@ -63072000 -63072000000 -63072000000000 +-63072000000000000 diff --git a/tests/queries/0_stateless/00538_datediff_plural_units.sql b/tests/queries/0_stateless/00538_datediff_plural_units.sql index d1234155a56..dd8395fc60f 100644 --- a/tests/queries/0_stateless/00538_datediff_plural_units.sql +++ b/tests/queries/0_stateless/00538_datediff_plural_units.sql @@ -8,3 +8,4 @@ SELECT dateDiff('minutes', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01- SELECT dateDiff('seconds', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01-01', 'UTC')); SELECT dateDiff('milliseconds', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01-01', 'UTC')); SELECT dateDiff('microseconds', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01-01', 'UTC')); +SELECT dateDiff('nanoseconds', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01-01', 'UTC')); diff --git a/tests/queries/0_stateless/02814_age_datediff.reference b/tests/queries/0_stateless/02814_age_datediff.reference index cbcb8c8a7b6..85dc205a499 100644 --- a/tests/queries/0_stateless/02814_age_datediff.reference +++ b/tests/queries/0_stateless/02814_age_datediff.reference @@ -1,10 +1,14 @@ -- { echo } -- DateTime64 vs DateTime64 with fractional part +SELECT age('nanosecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400005', 9, 'UTC')); +5100200000 +SELECT age('nanosecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400004', 9, 'UTC')); +5100199999 SELECT age('microsecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400005', 9, 'UTC')); 5100200 SELECT age('microsecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400004', 9, 'UTC')); -5100200 +5100199 SELECT age('millisecond', toDateTime64('2015-08-18 20:30:36.450299', 6, 'UTC'), toDateTime64('2015-08-18 20:30:41.550299', 6, 'UTC')); 5100 SELECT age('millisecond', toDateTime64('2015-08-18 20:30:36.450299', 6, 'UTC'), toDateTime64('2015-08-18 20:30:41.550298', 6, 'UTC')); diff --git a/tests/queries/0_stateless/02814_age_datediff.sql b/tests/queries/0_stateless/02814_age_datediff.sql index 934a95c035f..64e329b2fc5 100644 --- a/tests/queries/0_stateless/02814_age_datediff.sql +++ b/tests/queries/0_stateless/02814_age_datediff.sql @@ -1,6 +1,9 @@ -- { echo } -- DateTime64 vs DateTime64 with fractional part +SELECT age('nanosecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400005', 9, 'UTC')); +SELECT age('nanosecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400004', 9, 'UTC')); + SELECT age('microsecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400005', 9, 'UTC')); SELECT age('microsecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400004', 9, 'UTC')); From a05ca50b01abc21404252596f5c99a4a4d6684b1 Mon Sep 17 00:00:00 2001 From: Austin Kothig Date: Thu, 14 Mar 2024 17:48:54 -0400 Subject: [PATCH 0132/1165] Update docs/ru/sql-reference/functions/date-time-functions.md Co-authored-by: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> --- docs/ru/sql-reference/functions/date-time-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index 65c482e9914..56ae4359bf1 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -627,7 +627,7 @@ SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(d ## age -Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 наносекунда. +Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 наносекунду. Например, разница между `2021-12-29` и `2022-01-01` 3 дня для единицы `day`, 0 месяцев для единицы `month`, 0 лет для единицы `year`. **Синтаксис** From 25c56ac0a6c291b109bde878b1c3a85c2b1d91ca Mon Sep 17 00:00:00 2001 From: justindeguzman Date: Thu, 14 Mar 2024 18:30:18 -0700 Subject: [PATCH 0133/1165] [Docs] Specify that MySQL table engine is not supported in Cloud --- docs/en/engines/table-engines/integrations/mysql.md | 4 ++++ utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 2 files changed, 5 insertions(+) diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md index 87d59a9394b..978f1589e11 100644 --- a/docs/en/engines/table-engines/integrations/mysql.md +++ b/docs/en/engines/table-engines/integrations/mysql.md @@ -4,8 +4,12 @@ sidebar_position: 138 sidebar_label: MySQL --- +import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; + # MySQL + + The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server. ## Creating a Table {#creating-a-table} diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index cbd53e5dfce..e81ef772489 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -161,6 +161,7 @@ ClickHouseVapor ClickVisual ClickableSquare CloudDetails +CloudNotSupportedBadge CodeBlock CodeLLDB Codecs From 22e911dbe42219eb0cfb4fd403c4c520db1cd112 Mon Sep 17 00:00:00 2001 From: justindeguzman Date: Thu, 14 Mar 2024 18:56:28 -0700 Subject: [PATCH 0134/1165] [Docs] Specify that MySQL database engine is not supported in Cloud --- docs/en/engines/database-engines/mysql.md | 6 +++++- docs/en/engines/table-engines/integrations/mysql.md | 6 +++--- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/database-engines/mysql.md b/docs/en/engines/database-engines/mysql.md index 20434ad124e..ac0ea5e5af1 100644 --- a/docs/en/engines/database-engines/mysql.md +++ b/docs/en/engines/database-engines/mysql.md @@ -4,7 +4,11 @@ sidebar_position: 50 sidebar_label: MySQL --- -# MySQL +import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; + +# MySQL Database Engine + + Allows to connect to databases on a remote MySQL server and perform `INSERT` and `SELECT` queries to exchange data between ClickHouse and MySQL. diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md index 978f1589e11..e9d0a43242b 100644 --- a/docs/en/engines/table-engines/integrations/mysql.md +++ b/docs/en/engines/table-engines/integrations/mysql.md @@ -4,11 +4,11 @@ sidebar_position: 138 sidebar_label: MySQL --- -import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; +import CloudAvailableBadge from '@theme/badges/CloudAvailableBadge'; -# MySQL +# MySQL Table Engine - + The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server. diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index e81ef772489..f709d9b3546 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -160,6 +160,7 @@ ClickHouseNIO ClickHouseVapor ClickVisual ClickableSquare +CloudAvailableBadge CloudDetails CloudNotSupportedBadge CodeBlock From e5f0079fc66b06317fce92d0736c32a575f8aade Mon Sep 17 00:00:00 2001 From: peter279k Date: Fri, 15 Mar 2024 10:27:10 +0800 Subject: [PATCH 0135/1165] Correct the GEO datasets information --- docs/en/getting-started/example-datasets/cell-towers.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/getting-started/example-datasets/cell-towers.md b/docs/en/getting-started/example-datasets/cell-towers.md index 090de1b32fd..94fa6998f5d 100644 --- a/docs/en/getting-started/example-datasets/cell-towers.md +++ b/docs/en/getting-started/example-datasets/cell-towers.md @@ -75,14 +75,14 @@ This is the output of `DESCRIBE`. Down further in this guide the field type cho -1. Download the snapshot of the dataset from February 2021: [cell_towers.csv.xz](https://datasets.clickhouse.com/cell_towers.csv.xz) (729 MB). +1. Download the snapshot of the dataset from February 2021: [cell_towers.csv.xz](https://datasets.clickhouse.com/cell_towers.csv.xz) (686 MB). 2. Validate the integrity (optional step): ```bash md5sum cell_towers.csv.xz ``` ```response -8cf986f4a0d9f12c6f384a0e9192c908 cell_towers.csv.xz +8a797f7bdb55faba93f6cbc37d47b037 cell_towers.csv.xz ``` 3. Decompress it with the following command: @@ -132,7 +132,7 @@ SELECT radio, count() AS c FROM cell_towers GROUP BY radio ORDER BY c DESC ┌─radio─┬────────c─┐ │ UMTS │ 20686487 │ │ LTE │ 12101148 │ -│ GSM │ 9931312 │ +│ GSM │ 9931304 │ │ CDMA │ 556344 │ │ NR │ 867 │ └───────┴──────────┘ From f55e4f8f289e135ab65ca75fe6bb354b8cf00988 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 14 Mar 2024 09:51:02 +0100 Subject: [PATCH 0136/1165] Add test. --- .../test_dictionaries_replace/__init__.py | 0 .../configs/allow_database_types.xml | 7 + .../configs/no_lazy_load.xml | 3 + .../configs/remote_servers.xml | 16 +++ .../test_dictionaries_replace/test.py | 136 ++++++++++++++++++ 5 files changed, 162 insertions(+) create mode 100644 tests/integration/test_dictionaries_replace/__init__.py create mode 100644 tests/integration/test_dictionaries_replace/configs/allow_database_types.xml create mode 100644 tests/integration/test_dictionaries_replace/configs/no_lazy_load.xml create mode 100644 tests/integration/test_dictionaries_replace/configs/remote_servers.xml create mode 100644 tests/integration/test_dictionaries_replace/test.py diff --git a/tests/integration/test_dictionaries_replace/__init__.py b/tests/integration/test_dictionaries_replace/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dictionaries_replace/configs/allow_database_types.xml b/tests/integration/test_dictionaries_replace/configs/allow_database_types.xml new file mode 100644 index 00000000000..0434df06457 --- /dev/null +++ b/tests/integration/test_dictionaries_replace/configs/allow_database_types.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_dictionaries_replace/configs/no_lazy_load.xml b/tests/integration/test_dictionaries_replace/configs/no_lazy_load.xml new file mode 100644 index 00000000000..aaae3e0c4c1 --- /dev/null +++ b/tests/integration/test_dictionaries_replace/configs/no_lazy_load.xml @@ -0,0 +1,3 @@ + + 0 + diff --git a/tests/integration/test_dictionaries_replace/configs/remote_servers.xml b/tests/integration/test_dictionaries_replace/configs/remote_servers.xml new file mode 100644 index 00000000000..6362032973f --- /dev/null +++ b/tests/integration/test_dictionaries_replace/configs/remote_servers.xml @@ -0,0 +1,16 @@ + + + + + + node_ll + 9000 + + + node_no_ll + 9000 + + + + + diff --git a/tests/integration/test_dictionaries_replace/test.py b/tests/integration/test_dictionaries_replace/test.py new file mode 100644 index 00000000000..bf406f46cb1 --- /dev/null +++ b/tests/integration/test_dictionaries_replace/test.py @@ -0,0 +1,136 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + + +cluster = ClickHouseCluster(__file__) + +node_ll = cluster.add_instance( + "node_ll", + main_configs=[ + "configs/remote_servers.xml", + ], + user_configs=[ + "configs/allow_database_types.xml", + ], + macros={"replica": "node_ll", "shard": "shard"}, + with_zookeeper=True, +) + +node_no_ll = cluster.add_instance( + "node_no_ll", + main_configs=[ + "configs/no_lazy_load.xml", + "configs/remote_servers.xml", + ], + user_configs=[ + "configs/allow_database_types.xml", + ], + macros={"replica": "node_no_ll", "shard": "shard"}, + with_zookeeper=True, +) + +instances = [node_ll, node_no_ll] + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + # "sleep(1)" is here to make loading of dictionaries a bit slower for this test. + instances[0].query( + "CREATE VIEW src ON CLUSTER 'cluster' AS SELECT number AS key, number * number + 1 + sleep(1) AS value FROM numbers(10)" + ) + + # "dict_get_user" can only call dictGet(), nothing more. + instances[0].query("CREATE USER dictget_user ON CLUSTER 'cluster'") + instances[0].query( + "GRANT dictGet ON atomicdb.dict TO dictget_user ON CLUSTER 'cluster'" + ) + instances[0].query( + "GRANT dictGet ON repldb.dict TO dictget_user ON CLUSTER 'cluster'" + ) + + instances[0].query("CREATE DATABASE atomicdb ON CLUSTER 'cluster'") + instances[0].query( + "CREATE DATABASE repldb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}')" + ) + + yield cluster + + finally: + instances[0].query("DROP TABLE IF EXISTS src ON CLUSTER 'cluster'") + instances[0].query("DROP USER IF EXISTS dictget_user ON CLUSTER 'cluster'") + instances[0].query("DROP DATABASE IF EXISTS atomicdb ON CLUSTER 'cluster'") + instances[0].query("DROP DATABASE IF EXISTS repldb ON CLUSTER 'cluster'") + + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def cleanup_after_test(): + try: + yield + finally: + instances[0].query("DROP DICTIONARY IF EXISTS dict ON CLUSTER 'cluster'") + + +def get_status(instance, dictionary_name): + return instance.query( + "SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'" + ).rstrip("\n") + + +@pytest.mark.parametrize( + "database, instance_to_create_dictionary, instances_to_check", + [ + ("atomicdb", node_ll, [node_ll]), + ("atomicdb", node_no_ll, [node_no_ll]), + ("repldb", node_ll, [node_ll, node_no_ll]), + ("repldb", node_no_ll, [node_ll, node_no_ll]), + ], +) +def test_create_or_replace(database, instance_to_create_dictionary, instances_to_check): + num_steps = 2 + dict_uuids = {} + for step in range(0, num_steps): + create_dictionary_query = f"CREATE OR REPLACE DICTIONARY {database}.dict (key Int64, value Int64) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'src' PASSWORD '' DB 'default')) LIFETIME(0) LAYOUT(FLAT())" + instance_to_create_dictionary.query(create_dictionary_query) + + for instance in instances_to_check: + if instance != instance_to_create_dictionary: + instance.query(f"SYSTEM SYNC DATABASE REPLICA {database}") + + dict_uuid = instance.query( + f"SELECT uuid FROM system.tables WHERE database='{database}' AND table='dict'" + ).strip("\n") + + dict_db, dict_name, dict_status = ( + instance.query( + f"SELECT database, name, status FROM system.dictionaries WHERE uuid='{dict_uuid}'" + ) + .strip("\n") + .split("\t") + ) + assert dict_db == database + assert dict_name == "dict" + + # "uuid" must be the same for all the dictionaries created at the same "step" and different for the dictionaries created at different steps. + if step in dict_uuids: + assert dict_uuids[step] == dict_uuid + dict_uuids[step] = dict_uuid + assert dict_uuid not in [ + dict_uuids[prev_step] for prev_step in range(0, step) + ] + + expected_dict_status = ( + ["NOT_LOADED"] if instance == node_ll else ["LOADING", "LOADED"] + ) + assert dict_status in expected_dict_status + + for instance in instances_to_check: + select_query = f"SELECT arrayJoin([0, 5, 7, 11]) as key, dictGet({database}.dict, 'value', key)" + expected_result = TSV([[0, 1], [5, 26], [7, 50], [11, 0]]) + assert instance.query(select_query) == expected_result + assert instance.query(select_query, user="dictget_user") == expected_result From 00533f3df634c3a96e78fa0732c9375f257ffb5b Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 15 Mar 2024 19:43:44 +0800 Subject: [PATCH 0137/1165] revert opts in column array --- src/Columns/ColumnArray.cpp | 232 +++--------------------------------- src/Columns/ColumnArray.h | 14 +-- 2 files changed, 19 insertions(+), 227 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 0214375122f..7b268b80116 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -31,7 +31,6 @@ namespace ErrorCodes extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; extern const int LOGICAL_ERROR; extern const int TOO_LARGE_ARRAY_SIZE; - extern const int ILLEGAL_COLUMN; } /** Obtaining array as Field can be slow for large arrays and consume vast amount of memory. @@ -43,34 +42,29 @@ namespace ErrorCodes static constexpr size_t max_array_size_as_field = 1000000; -ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column, bool check_offsets) +ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column) : data(std::move(nested_column)), offsets(std::move(offsets_column)) { - if (check_offsets) + const ColumnOffsets * offsets_concrete = typeid_cast(offsets.get()); + + if (!offsets_concrete) + throw Exception(ErrorCodes::LOGICAL_ERROR, "offsets_column must be a ColumnUInt64"); + + if (!offsets_concrete->empty() && data && !data->empty()) { - const ColumnOffsets * offsets_concrete = typeid_cast(offsets.get()); + Offset last_offset = offsets_concrete->getData().back(); - if (!offsets_concrete) - throw Exception(ErrorCodes::LOGICAL_ERROR, "offsets_column must be a ColumnUInt64"); - - if (!offsets_concrete->empty() && data && !data->empty()) - { - Offset last_offset = offsets_concrete->getData().back(); - - /// This will also prevent possible overflow in offset. - if (data->size() != last_offset) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "offsets_column has data inconsistent with nested_column. Data size: {}, last offset: {}", - data->size(), - last_offset); - } - - /** NOTE - * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). - * But in most cases, arrays with constant value are unexpected and code will work wrong. Use with caution. - */ + /// This will also prevent possible overflow in offset. + if (data->size() != last_offset) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "offsets_column has data inconsistent with nested_column. Data size: {}, last offset: {}", + data->size(), last_offset); } + + /** NOTE + * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). + * But in most cases, arrays with constant value are unexpected and code will work wrong. Use with caution. + */ } ColumnArray::ColumnArray(MutableColumnPtr && nested_column) @@ -353,196 +347,6 @@ void ColumnArray::insertFrom(const IColumn & src_, size_t n) getOffsets().push_back(getOffsets().back() + size); } -template -void ColumnArray::insertManyFromNumber(const ColumnArray & src, size_t position, size_t length) -{ - using ColVecType = ColumnVectorOrDecimal; - size_t src_size = src.sizeAt(position); - size_t src_offset = src.offsetAt(position); - - const typename ColVecType::Container & src_data = typeid_cast(src.getData()).getData(); - typename ColVecType::Container & data_ref = typeid_cast(getData()).getData(); - size_t old_size = data_ref.size(); - size_t new_size = old_size + src_size * length; - data_ref.resize(new_size); - for (size_t i = 0, offset = old_size; i < length; ++i, offset += src_size) - memcpy(&data_ref[offset], &src_data[src_offset], src_size * sizeof(T)); -} - -void ColumnArray::insertManyFromConst(const ColumnConst & src, size_t /*position*/, size_t length) -{ - const ColumnArray * src_array = typeid_cast(&src.getDataColumn()); - if (!src_array) - throw Exception( - ErrorCodes::ILLEGAL_COLUMN, - "Cannot insert from const column of type {} to column of type {}", - src.getDataColumn().getName(), - getName()); - - insertManyFromImpl(*src_array, 0, length, true); -} - -void ColumnArray::insertManyFromString(const ColumnArray & src, size_t position, size_t length) -{ - size_t src_size = src.sizeAt(position); - size_t src_offset = src.offsetAt(position); - - const auto & src_string = typeid_cast(src.getData()); - const auto & src_chars = src_string.getChars(); - const auto & src_string_offsets = src_string.getOffsets(); - auto & dst_string = typeid_cast(getData()); - auto & dst_chars = dst_string.getChars(); - auto & dst_string_offsets = dst_string.getOffsets(); - - /// Each row may have multiple strings, copy them to dst_chars and update dst_offsets - size_t old_size = dst_string_offsets.size(); - size_t new_size = old_size + src_size * length; - dst_string_offsets.resize(new_size); - size_t dst_string_offset = dst_chars.size(); - for (size_t i = 0; i < length; ++i) - { - for (size_t j = 0; j < src_size; ++j) - { - size_t nested_offset = src_string_offsets[src_offset + j - 1]; - size_t nested_length = src_string_offsets[src_offset + j] - nested_offset; - - dst_string_offset += nested_length; - dst_string_offsets[old_size + i * src_size + j] = dst_string_offset; - } - } - - size_t chars_to_copy = src_string_offsets[src_offset + src_size - 1] - src_string_offsets[src_offset - 1]; - dst_chars.resize(dst_chars.size() + chars_to_copy * length); - for (size_t dst_offset = old_size; dst_offset < new_size; dst_offset += src_size) - memcpy(&dst_chars[dst_string_offsets[dst_offset - 1]], &src_chars[src_string_offsets[src_offset - 1]], chars_to_copy); -} - -void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, size_t length) -{ - ColumnTuple & tuple = assert_cast(getData()); - const ColumnTuple & src_tuple = assert_cast(src.getData()); - - /// Make temporary arrays for each components of Tuple. In the same way as for Nullable. - size_t tuple_size = tuple.tupleSize(); - size_t src_tuple_size = src_tuple.tupleSize(); - if (tuple_size == 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty tuple"); - if (tuple_size != src_tuple_size) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Nested tuple size mismatch: {} vs {}", tuple_size, src_tuple_size); - - Columns tuple_columns(tuple_size); - for (size_t i = 0; i < tuple_size; ++i) - { - auto array_of_element = ColumnArray(tuple.getColumn(i).assumeMutable(), getOffsetsPtr()->assumeMutable(), false); - auto src_array_of_element = ColumnArray(src_tuple.getColumn(i).assumeMutable(), src.getOffsetsPtr()->assumeMutable()); - array_of_element.insertManyFromImpl(src_array_of_element, position, length, false); - tuple_columns[i] = array_of_element.getDataPtr(); - } - getDataPtr() = ColumnTuple::create(std::move(tuple_columns)); -} - -void ColumnArray::insertManyFromNullable(const ColumnArray & src, size_t position, size_t length) -{ - ColumnNullable & nullable = assert_cast(getData()); - const ColumnNullable & src_nullable = assert_cast(src.getData()); - - /// Process nested column without updating array offsets - auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable(), false); - auto src_array_of_nested = ColumnArray(src_nullable.getNestedColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); - array_of_nested.insertManyFromImpl(src_array_of_nested, position, length, false); - - /// Process null map column without updating array offsets - auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable(), false); - auto src_array_of_null_map = ColumnArray(src_nullable.getNullMapColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); - array_of_null_map.insertManyFromImpl(src_array_of_null_map, position, length, false); - - /// Update array data - getDataPtr() = ColumnNullable::create(array_of_nested.getDataPtr(), array_of_null_map.getDataPtr()); -} - -void ColumnArray::insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length) -{ - size_t src_size = src.sizeAt(position); - size_t src_offset = src.offsetAt(position); - const auto & src_data = src.getData(); - size_t new_size = data->size() + src_size * length; - data->reserve(new_size); - for (size_t i = 0; i < length; ++i) - data->insertRangeFrom(src_data, src_offset, src_size); -} - -void ColumnArray::insertManyFrom(const IColumn & src_, size_t position, size_t length) -{ - const ColumnConst * src_const = typeid_cast(&src_); - if (src_const) - return insertManyFromConst(*src_const, position, length); - - const ColumnArray * src_array = typeid_cast(&src_); - if (!src_array) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert from column of type {} to column of type {}", src_.getName(), getName()); - - return insertManyFromImpl(*src_array, position, length, true); -} - -void ColumnArray::insertManyFromImpl(const ColumnArray & src, size_t position, size_t length, bool update_offsets) -{ - /// First fill offsets if needed - if (update_offsets) - { - size_t src_size = src.sizeAt(position); - auto & offsets_ref = getOffsets(); - size_t old_rows = offsets_ref.size(); - size_t new_rows = old_rows + length; - size_t old_size = offsets_ref.back(); - offsets_ref.resize(new_rows); - for (size_t i = 0, offset = old_size + src_size; i < length; ++i, offset += src_size) - offsets_ref[old_rows + i] = offset; - } - - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromString(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNullable(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromTuple(src, position, length); - return insertManyFromGeneric(src, position, length); -} void ColumnArray::insertDefault() { diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 8c4d103e7d0..230d8830265 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -21,7 +21,7 @@ private: friend class COWHelper, ColumnArray>; /** Create an array column with specified values and offsets. */ - ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column, bool check_offsets = true); + ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column); /** Create an empty column of arrays with the type of values as in the column `nested_column` */ explicit ColumnArray(MutableColumnPtr && nested_column); @@ -88,7 +88,6 @@ public: void insert(const Field & x) override; bool tryInsert(const Field & x) override; void insertFrom(const IColumn & src_, size_t n) override; - void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertDefault() override; void popBack(size_t n) override; ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; @@ -214,17 +213,6 @@ private: ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const; ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const; - /// Specializations for insertManyFrom - void insertManyFromConst(const ColumnConst & src, size_t position, size_t length); - void insertManyFromImpl(const ColumnArray & src, size_t position, size_t length, bool update_offsets = true); - - template - void insertManyFromNumber(const ColumnArray & src, size_t position, size_t length); - void insertManyFromString(const ColumnArray & src, size_t position, size_t length); - void insertManyFromTuple(const ColumnArray & src, size_t position, size_t length); - void insertManyFromNullable(const ColumnArray & src, size_t position, size_t length); - void insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length); - int compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint, const Collator * collator=nullptr) const; }; From 4285f1a8114084b0b7af8dd3546eae1953072915 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 15 Mar 2024 13:50:03 +0100 Subject: [PATCH 0138/1165] Revert "Revert "Updated format settings references in the docs (datetime.md)"" --- docs/en/sql-reference/data-types/datetime.md | 8 ++++---- docs/ru/sql-reference/data-types/datetime.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/data-types/datetime.md b/docs/en/sql-reference/data-types/datetime.md index 1adff18f598..a465106c2ff 100644 --- a/docs/en/sql-reference/data-types/datetime.md +++ b/docs/en/sql-reference/data-types/datetime.md @@ -36,9 +36,9 @@ You can explicitly set a time zone for `DateTime`-type columns when creating a t The [clickhouse-client](../../interfaces/cli.md) applies the server time zone by default if a time zone isn’t explicitly set when initializing the data type. To use the client time zone, run `clickhouse-client` with the `--use_client_time_zone` parameter. -ClickHouse outputs values depending on the value of the [date_time_output_format](../../operations/settings/settings.md#settings-date_time_output_format) setting. `YYYY-MM-DD hh:mm:ss` text format by default. Additionally, you can change the output with the [formatDateTime](../../sql-reference/functions/date-time-functions.md#formatdatetime) function. +ClickHouse outputs values depending on the value of the [date_time_output_format](../../operations/settings/settings-formats.md#date_time_output_format) setting. `YYYY-MM-DD hh:mm:ss` text format by default. Additionally, you can change the output with the [formatDateTime](../../sql-reference/functions/date-time-functions.md#formatdatetime) function. -When inserting data into ClickHouse, you can use different formats of date and time strings, depending on the value of the [date_time_input_format](../../operations/settings/settings.md#settings-date_time_input_format) setting. +When inserting data into ClickHouse, you can use different formats of date and time strings, depending on the value of the [date_time_input_format](../../operations/settings/settings-formats.md#date_time_input_format) setting. ## Examples @@ -147,8 +147,8 @@ Time shifts for multiple days. Some pacific islands changed their timezone offse - [Type conversion functions](../../sql-reference/functions/type-conversion-functions.md) - [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md) - [Functions for working with arrays](../../sql-reference/functions/array-functions.md) -- [The `date_time_input_format` setting](../../operations/settings/settings-formats.md#settings-date_time_input_format) -- [The `date_time_output_format` setting](../../operations/settings/settings-formats.md#settings-date_time_output_format) +- [The `date_time_input_format` setting](../../operations/settings/settings-formats.md#date_time_input_format) +- [The `date_time_output_format` setting](../../operations/settings/settings-formats.md#date_time_output_format) - [The `timezone` server configuration parameter](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) - [The `session_timezone` setting](../../operations/settings/settings.md#session_timezone) - [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime) diff --git a/docs/ru/sql-reference/data-types/datetime.md b/docs/ru/sql-reference/data-types/datetime.md index 57f24786bb7..25e87794147 100644 --- a/docs/ru/sql-reference/data-types/datetime.md +++ b/docs/ru/sql-reference/data-types/datetime.md @@ -27,9 +27,9 @@ DateTime([timezone]) Консольный клиент ClickHouse по умолчанию использует часовой пояс сервера, если для значения `DateTime` часовой пояс не был задан в явном виде при инициализации типа данных. Чтобы использовать часовой пояс клиента, запустите [clickhouse-client](../../interfaces/cli.md) с параметром `--use_client_time_zone`. -ClickHouse отображает значения в зависимости от значения параметра [date\_time\_output\_format](../../operations/settings/index.md#settings-date_time_output_format). Текстовый формат по умолчанию `YYYY-MM-DD hh:mm:ss`. Кроме того, вы можете поменять отображение с помощью функции [formatDateTime](../../sql-reference/functions/date-time-functions.md#formatdatetime). +ClickHouse отображает значения в зависимости от значения параметра [date\_time\_output\_format](../../operations/settings/settings-formats.md#date_time_output_format). Текстовый формат по умолчанию `YYYY-MM-DD hh:mm:ss`. Кроме того, вы можете поменять отображение с помощью функции [formatDateTime](../../sql-reference/functions/date-time-functions.md#formatdatetime). -При вставке данных в ClickHouse, можно использовать различные форматы даты и времени в зависимости от значения настройки [date_time_input_format](../../operations/settings/index.md#settings-date_time_input_format). +При вставке данных в ClickHouse, можно использовать различные форматы даты и времени в зависимости от значения настройки [date_time_input_format](../../operations/settings/settings-formats.md#date_time_input_format). ## Примеры {#primery} @@ -119,8 +119,8 @@ FROM dt - [Функции преобразования типов](../../sql-reference/functions/type-conversion-functions.md) - [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md) - [Функции для работы с массивами](../../sql-reference/functions/array-functions.md) -- [Настройка `date_time_input_format`](../../operations/settings/index.md#settings-date_time_input_format) -- [Настройка `date_time_output_format`](../../operations/settings/index.md) +- [Настройка `date_time_input_format`](../../operations/settings/settings-formats.md#date_time_input_format) +- [Настройка `date_time_output_format`](../../operations/settings/settings-formats.md#date_time_output_format) - [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) - [Параметр `session_timezone`](../../operations/settings/settings.md#session_timezone) - [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime) From 7d38789e3053b6912300e9fd48456cd3b3426690 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 15 Mar 2024 14:43:01 +0000 Subject: [PATCH 0139/1165] Make generate_series test smaller --- tests/queries/0_stateless/02970_generate_series.reference | 5 +++-- tests/queries/0_stateless/02970_generate_series.sql | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02970_generate_series.reference b/tests/queries/0_stateless/02970_generate_series.reference index 4e4f556a39b..6e6f3c81587 100644 --- a/tests/queries/0_stateless/02970_generate_series.reference +++ b/tests/queries/0_stateless/02970_generate_series.reference @@ -2,8 +2,9 @@ 1 4 8 -500000001 -50000000 +501 +50 +17928 0 10 13 diff --git a/tests/queries/0_stateless/02970_generate_series.sql b/tests/queries/0_stateless/02970_generate_series.sql index a7f89e1bd3f..0844714b3a6 100644 --- a/tests/queries/0_stateless/02970_generate_series.sql +++ b/tests/queries/0_stateless/02970_generate_series.sql @@ -2,8 +2,9 @@ SELECT count() FROM generate_series(5, 4); SELECT count() FROM generate_series(0, 0); SELECT count() FROM generate_series(10, 20, 3); SELECT count() FROM generate_series(7, 77, 10); -SELECT count() FROM generate_series(0, 1000000000, 2); -SELECT count() FROM generate_series(0, 999999999, 20); +SELECT count() FROM generate_series(0, 1000, 2); +SELECT count() FROM generate_series(0, 999, 20); +SELECT sum(generate_series) FROM generate_series(4, 1008, 4) WHERE generate_series % 7 = 1; SELECT * FROM generate_series(5, 4); SELECT * FROM generate_series(0, 0); From 7c49d5bbf673b5ca855bae79609c6a2ea5bc2079 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 15 Mar 2024 15:35:31 +0000 Subject: [PATCH 0140/1165] Fix some tests #ci_set_analyzer --- .../ClusterProxy/SelectStreamFactory.cpp | 8 +------- .../OptimizeShardingKeyRewriteInVisitor.cpp | 8 +++++++- .../QueryPlan/DistributedCreateLocalPlan.cpp | 13 ++++++++----- .../QueryPlan/DistributedCreateLocalPlan.h | 3 ++- src/Processors/QueryPlan/ReadFromRemote.cpp | 6 +++++- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index ab301e01d0a..94a02eacdaa 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -157,14 +157,8 @@ void SelectStreamFactory::createForShardImpl( { auto emplace_local_stream = [&]() { - Block shard_header; - if (context->getSettingsRef().allow_experimental_analyzer) - shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_tree, context, SelectQueryOptions(processed_stage).analyze()); - else - shard_header = header; - local_plans.emplace_back(createLocalPlan( - query_ast, shard_header, context, processed_stage, shard_info.shard_num, shard_count)); + query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, has_missing_objects)); }; auto emplace_remote_stream = [&](bool lazy = false, time_t local_delay = 0) diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index e0d0fda81cb..0df6fcee229 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -11,6 +11,7 @@ #include "Analyzer/FunctionNode.h" #include "Analyzer/IQueryTreeNode.h" #include "Analyzer/InDepthQueryTreeVisitor.h" +#include "Analyzer/Utils.h" #include "DataTypes/IDataType.h" #include "Interpreters/Context_fwd.h" @@ -170,7 +171,12 @@ public: if (new_tuple.empty()) new_tuple.push_back(tuple.back()); - node = std::make_shared(new_tuple); + + if (new_tuple.size() == tuple.size()) + return; + + arguments[1] = std::make_shared(new_tuple); + rerunFunctionResolve(function_node, getContext()); } } } diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index 4d4bc6220e9..c8d230c87d9 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -14,12 +14,14 @@ namespace DB namespace { -void addConvertingActions(QueryPlan & plan, const Block & header) +void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects) { if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header)) return; - auto get_converting_dag = [](const Block & block_, const Block & header_) + auto mode = has_missing_objects ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; + + auto get_converting_dag = [mode](const Block & block_, const Block & header_) { /// Convert header structure to expected. /// Also we ignore constants from result and replace it with constants from header. @@ -27,7 +29,7 @@ void addConvertingActions(QueryPlan & plan, const Block & header) return ActionsDAG::makeConvertingActions( block_.getColumnsWithTypeAndName(), header_.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name, + mode, true); }; @@ -44,7 +46,8 @@ std::unique_ptr createLocalPlan( ContextPtr context, QueryProcessingStage::Enum processed_stage, size_t shard_num, - size_t shard_count) + size_t shard_count, + bool has_missing_objects) { checkStackSize(); @@ -74,7 +77,7 @@ std::unique_ptr createLocalPlan( interpreter.buildQueryPlan(*query_plan); } - addConvertingActions(*query_plan, header); + addConvertingActions(*query_plan, header, has_missing_objects); return query_plan; } diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.h b/src/Processors/QueryPlan/DistributedCreateLocalPlan.h index 489d2607849..50545d9ae81 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.h +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.h @@ -18,5 +18,6 @@ std::unique_ptr createLocalPlan( ContextPtr context, QueryProcessingStage::Enum processed_stage, size_t shard_num, - size_t shard_count); + size_t shard_count, + bool has_missing_objects); } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 8c455883ab2..119710d06d8 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +51,9 @@ static void addConvertingActions(Pipe & pipe, const Block & header, bool use_pos true); }; + if (use_positions_to_match) + pipe.addSimpleTransform([](const Block & stream_header) { return std::make_shared(stream_header); }); + auto convert_actions = std::make_shared(get_converting_dag(pipe.getHeader(), header)); pipe.addSimpleTransform([&](const Block & cur_header, Pipe::StreamType) -> ProcessorPtr { @@ -190,7 +194,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream if (try_results.empty() || local_delay < max_remote_delay) { auto plan = createLocalPlan( - query, header, my_context, my_stage, my_shard.shard_info.shard_num, my_shard_count); + query, header, my_context, my_stage, my_shard.shard_info.shard_num, my_shard_count, my_shard.has_missing_objects); return std::move(*plan->buildQueryPipeline( QueryPlanOptimizationSettings::fromContext(my_context), From db0a5209f1ed0ddc88057ce8d2425b97e9c84397 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 15 Mar 2024 16:14:59 +0000 Subject: [PATCH 0141/1165] Fix tests --- .../02783_parallel_replicas_trivial_count_optimization.sh | 4 ++-- .../0_stateless/03001_max_parallel_replicas_zero_value.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh b/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh index bafab249b47..20b3efedd49 100755 --- a/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh +++ b/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh @@ -25,7 +25,7 @@ function run_query_with_pure_parallel_replicas () { $CLICKHOUSE_CLIENT \ --query "$2" \ --query_id "${1}_disabled" \ - --max_parallel_replicas 0 + --max_parallel_replicas 1 $CLICKHOUSE_CLIENT \ --query "$2" \ @@ -50,7 +50,7 @@ function run_query_with_custom_key_parallel_replicas () { $CLICKHOUSE_CLIENT \ --query "$2" \ --query_id "${1}_disabled" \ - --max_parallel_replicas 0 + --max_parallel_replicas 1 $CLICKHOUSE_CLIENT \ --query "$2" \ diff --git a/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql b/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql index 611aa4777ba..499486713a6 100644 --- a/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql +++ b/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql @@ -1,5 +1,5 @@ drop table if exists test_d; create table test_d engine=Distributed(test_cluster_two_shard_three_replicas_localhost, system, numbers); -select * from test_d limit 10 settings max_parallel_replicas = 0, prefer_localhost_replica = 0; --{serverError BAD_ARGUMENTS} +select * from test_d limit 10 settings max_parallel_replicas = 0, prefer_localhost_replica = 0; --{clientError BAD_ARGUMENTS} drop table test_d; From fa9d9ea3f74e1f813fd40614dc5944f189bb9d94 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 15 Mar 2024 20:23:39 +0000 Subject: [PATCH 0142/1165] Ignore DROP queries in stress test with 1/2 probability, use TRUNCATE instead of ignoring DROP in upgrade check for Memory/JOIN tables --- programs/client/Client.cpp | 27 +++++---------- src/Client/ClientBase.cpp | 67 +++++++++++++++++++++++++++++++++++++- src/Client/ClientBase.h | 9 ++++- tests/ci/stress.py | 5 +++ tests/clickhouse-test | 2 ++ 5 files changed, 89 insertions(+), 21 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 4203e4738dd..70550b5952a 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -50,7 +50,6 @@ #include #include #include -#include namespace fs = std::filesystem; using namespace std::literals; @@ -953,7 +952,9 @@ void Client::addOptions(OptionsDescription & options_description) ("opentelemetry-tracestate", po::value(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation") ("no-warnings", "disable warnings when client connects to server") + /// TODO: Left for compatibility as it's used in upgrade check, remove after next release and use ignore-drop-queries-probability ("fake-drop", "Ignore all DROP queries, should be used only for testing") + ("ignore-drop-queries-probability", po::value(), "With specified probability ignore all DROP queries (replace them to TRUNCATE for engines like Memory/JOIN), should be used only for testing") ("accept-invalid-certificate", "Ignore certificate verification errors, equal to config parameters openSSL.client.invalidCertificateHandler.name=AcceptCertificateHandler and openSSL.client.verificationMode=none") ; @@ -1096,7 +1097,9 @@ void Client::processOptions(const OptionsDescription & options_description, if (options.count("no-warnings")) config().setBool("no-warnings", true); if (options.count("fake-drop")) - fake_drop = true; + ignore_drop_queries_probability = 1; + if (options.count("ignore-drop-queries-probability")) + ignore_drop_queries_probability = std::min(options["ignore-drop-queries-probability"].as(), 1.); if (options.count("accept-invalid-certificate")) { config().setString("openSSL.client.invalidCertificateHandler.name", "AcceptCertificateHandler"); @@ -1138,13 +1141,6 @@ void Client::processOptions(const OptionsDescription & options_description, } -static bool checkIfStdoutIsRegularFile() -{ - struct stat file_stat; - return fstat(STDOUT_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode); -} - - void Client::processConfig() { if (!queries.empty() && config().has("queries-file")) @@ -1181,14 +1177,7 @@ void Client::processConfig() pager = config().getString("pager", ""); is_default_format = !config().has("vertical") && !config().has("format"); - if (is_default_format && checkIfStdoutIsRegularFile()) - { - is_default_format = false; - std::optional format_from_file_name; - format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDOUT_FILENO); - format = format_from_file_name ? *format_from_file_name : "TabSeparated"; - } - else if (config().has("vertical")) + if (config().has("vertical")) format = config().getString("format", "Vertical"); else format = config().getString("format", is_interactive ? "PrettyCompact" : "TabSeparated"); @@ -1392,8 +1381,8 @@ void Client::readArguments( } -#pragma clang diagnostic ignored "-Wunused-function" -#pragma clang diagnostic ignored "-Wmissing-declarations" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" int mainEntryClickHouseClient(int argc, char ** argv) { diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 48962880b8f..c0865d4fb13 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -68,6 +68,9 @@ #include #include +#include +#include + #include #include #include @@ -559,6 +562,11 @@ try pager_cmd = ShellCommand::execute(config); out_buf = &pager_cmd->in; } + /// We can use special buffer for query output for internal queries. + else if (output_format_buffer) + { + out_buf = output_format_buffer.get(); + } else { out_buf = &std_out; @@ -868,11 +876,68 @@ void ClientBase::processTextAsSingleQuery(const String & full_query) processError(full_query); } +String ClientBase::getTableEngine(const String & database, const String & table) +{ + auto is_interactive_copy = is_interactive; + auto format_copy = format; + + is_interactive = false; + format = "TSVRaw"; + String result; + output_format_buffer = std::make_unique(result); + String query; + if (database.empty()) + query = fmt::format("SELECT engine FROM system.tables where name='{}' and database=currentDatabase()", table); + else + query = fmt::format("SELECT engine FROM system.tables where name='{}' and database='{}'", table, database); + + try + { + processTextAsSingleQuery(query); + } + catch (...) + { + result = ""; + } + + output_format_buffer->finalize(); + output_format_buffer.reset(); + is_interactive = is_interactive_copy; + format = format_copy; + boost::trim(result); + return result; +} + +void ClientBase::ignoreDropQueryOrTruncateTable(const DB::ASTDropQuery * drop_query) +{ + const auto & database = drop_query->getDatabase(); + const auto & table = drop_query->getTable(); + /// Use TRUNCATE for Memory/JOIN table engines to reduce memory usage in tests. + String table_engine = getTableEngine(database, table); + if (table_engine == "Memory" || table_engine == "JOIN") + { + String truncate_query; + if (database.empty()) + truncate_query = fmt::format("TRUNCATE TABLE {}", drop_query->getTable()); + else + truncate_query = fmt::format("TRUNCATE TABLE {}.{}", drop_query->getDatabase(), drop_query->getTable()); + + auto is_interactive_copy = is_interactive; + is_interactive = false; + processTextAsSingleQuery(truncate_query); + is_interactive = is_interactive_copy; + } +} void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr parsed_query) { - if (fake_drop && parsed_query->as()) + /// In tests we can ignore DROP queries with some probability. + const auto * drop_query = parsed_query->as(); + if (ignore_drop_queries_probability != 0 && drop_query && drop_query->kind == ASTDropQuery::Kind::Drop && std::uniform_real_distribution<>(0.0, 1.0)(thread_local_rng) <= ignore_drop_queries_probability) + { + ignoreDropQueryOrTruncateTable(drop_query); return; + } auto query = query_to_execute; diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index dd08e7c059b..fd81c4c4080 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -178,6 +178,12 @@ private: void initQueryIdFormats(); bool addMergeTreeSettings(ASTCreateQuery & ast_create); + void ignoreDropQueryOrTruncateTable(const ASTDropQuery * drop_query); + /// Request table engine from system.tables from server. + String getTableEngine(const String & database, const String & table); + /// Send TRUNCATE query for specific table. + void truncateTable(const String & database, const String & table); + protected: static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context); bool processMultiQueryFromFile(const String & file_name); @@ -248,6 +254,7 @@ protected: /// The user can specify to redirect query output to a file. std::unique_ptr out_file_buf; std::shared_ptr output_format; + std::unique_ptr output_format_buffer; /// The user could specify special file for server logs (stderr by default) std::unique_ptr out_logs_buf; @@ -307,7 +314,7 @@ protected: QueryProcessingStage::Enum query_processing_stage; ClientInfo::QueryKind query_kind; - bool fake_drop = false; + double ignore_drop_queries_probability = 0; struct HostAndPort { diff --git a/tests/ci/stress.py b/tests/ci/stress.py index 7ccc058f79f..b0076449a53 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -66,6 +66,11 @@ def get_options(i: int, upgrade_check: bool) -> str: if random.random() < 0.3: client_options.append(f"http_make_head_request={random.randint(0, 1)}") + # TODO: After release 23.3 use ignore-drop-queries-probability for both + # stress test and upgrade check + if not upgrade_check: + client_options.append("ignore-drop-queries-probability=0.5") + if client_options: options.append(" --client-option " + " ".join(client_options)) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index ce0feadf050..cc62a180579 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -2872,6 +2872,8 @@ def parse_args(): help="Do not run shard related tests", ) + # TODO: Remove upgrade-check option after release 23.3 and use + # ignore-drop-queries-probability option in stress.py as in stress tests group.add_argument( "--upgrade-check", action="store_true", From 4ceff16787eb16f5b485a6d70f6d8b29744a16bd Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 15 Mar 2024 20:29:19 +0000 Subject: [PATCH 0143/1165] Fix bad conflict resolution --- programs/client/Client.cpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 70550b5952a..caf2ce2f68a 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -50,6 +50,7 @@ #include #include #include +#include namespace fs = std::filesystem; using namespace std::literals; @@ -1141,6 +1142,13 @@ void Client::processOptions(const OptionsDescription & options_description, } +static bool checkIfStdoutIsRegularFile() +{ + struct stat file_stat; + return fstat(STDOUT_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode); +} + + void Client::processConfig() { if (!queries.empty() && config().has("queries-file")) @@ -1177,7 +1185,14 @@ void Client::processConfig() pager = config().getString("pager", ""); is_default_format = !config().has("vertical") && !config().has("format"); - if (config().has("vertical")) + if (is_default_format && checkIfStdoutIsRegularFile()) + { + is_default_format = false; + std::optional format_from_file_name; + format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDOUT_FILENO); + format = format_from_file_name ? *format_from_file_name : "TabSeparated"; + } + else if (config().has("vertical")) format = config().getString("format", "Vertical"); else format = config().getString("format", is_interactive ? "PrettyCompact" : "TabSeparated"); @@ -1381,8 +1396,8 @@ void Client::readArguments( } -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wmissing-declarations" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wmissing-declarations" int mainEntryClickHouseClient(int argc, char ** argv) { From 2e9130ca1e99587e47fcfff4be318601e4d52cfc Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 15 Mar 2024 20:30:44 +0000 Subject: [PATCH 0144/1165] Better --- src/Client/ClientBase.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index c0865d4fb13..0fb8b27e20f 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -918,9 +918,9 @@ void ClientBase::ignoreDropQueryOrTruncateTable(const DB::ASTDropQuery * drop_qu { String truncate_query; if (database.empty()) - truncate_query = fmt::format("TRUNCATE TABLE {}", drop_query->getTable()); + truncate_query = fmt::format("TRUNCATE TABLE {}", table); else - truncate_query = fmt::format("TRUNCATE TABLE {}.{}", drop_query->getDatabase(), drop_query->getTable()); + truncate_query = fmt::format("TRUNCATE TABLE {}.{}", database, table); auto is_interactive_copy = is_interactive; is_interactive = false; From adaf262dc09ff08628d54d5ffd64f7ea5053482d Mon Sep 17 00:00:00 2001 From: Dan Wu Date: Sat, 16 Mar 2024 01:12:59 +0000 Subject: [PATCH 0145/1165] Fix bug when reading system.parts using UUID (issue 61220). During the predicates pushing down of system.parts table, there is a logic to use that predicates to filter on the storages. This makes sense if the predicate is on the `database`, `table`, `engine`. But it can be problem if the predicate is about `uuid`, because storage UUID is apparently different from parts UUID. Rename the column name from `uuid` to `storage_uuid` fixed this. --- src/Storages/System/StorageSystemPartsBase.cpp | 4 ++-- .../03010_read_system_parts_table_test.reference | 4 ++++ .../0_stateless/03010_read_system_parts_table_test.sql | 10 ++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03010_read_system_parts_table_test.reference create mode 100644 tests/queries/0_stateless/03010_read_system_parts_table_test.sql diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 6bdfdd357e8..1283f2d6ce0 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -188,7 +188,7 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, Conte block_to_filter.insert(ColumnWithTypeAndName(std::move(table_column_mut), std::make_shared(), "table")); block_to_filter.insert(ColumnWithTypeAndName(std::move(engine_column_mut), std::make_shared(), "engine")); block_to_filter.insert(ColumnWithTypeAndName(std::move(active_column_mut), std::make_shared(), "active")); - block_to_filter.insert(ColumnWithTypeAndName(std::move(storage_uuid_column_mut), std::make_shared(), "uuid")); + block_to_filter.insert(ColumnWithTypeAndName(std::move(storage_uuid_column_mut), std::make_shared(), "storage_uuid")); if (rows) { @@ -200,7 +200,7 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, Conte database_column = block_to_filter.getByName("database").column; table_column = block_to_filter.getByName("table").column; active_column = block_to_filter.getByName("active").column; - storage_uuid_column = block_to_filter.getByName("uuid").column; + storage_uuid_column = block_to_filter.getByName("storage_uuid").column; } class ReadFromSystemPartsBase : public SourceStepWithFilter diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.reference b/tests/queries/0_stateless/03010_read_system_parts_table_test.reference new file mode 100644 index 00000000000..6847786761e --- /dev/null +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.reference @@ -0,0 +1,4 @@ +00000000-0000-0000-0000-000000000000 1231_1_1_0 +00000000-0000-0000-0000-000000000000 6666_2_2_0 +users +users diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql new file mode 100644 index 00000000000..6ec7e6212d5 --- /dev/null +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS users; +CREATE TABLE users (uid Int16, name String, age Int16) ENGINE=MergeTree ORDER BY uid PARTITION BY uid; + +INSERT INTO users VALUES (1231, 'John', 33); +INSERT INTO users VALUES (6666, 'Ksenia', 48); + +SELECT uuid, name from system.parts WHERE table = 'users'; + +SELECT table from system.parts WHERE uuid = '00000000-0000-0000-0000-000000000000'; +DROP TABLE IF EXISTS users; From 335f147a3b6e69d7924a2f70449f5b2ee125bac6 Mon Sep 17 00:00:00 2001 From: Nikolay Yankin <211292+kolya7k@users.noreply.github.com> Date: Sat, 16 Mar 2024 16:57:39 +0300 Subject: [PATCH 0146/1165] Update install.md Alternative URL for latest version --- docs/en/getting-started/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 3b01434ecc5..cf95560b311 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -262,7 +262,7 @@ The required version can be downloaded with `curl` or `wget` from repository htt After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest stable version: ``` bash -LATEST_VERSION=$(curl -s -L https://api.github.com/repos/ClickHouse/ClickHouse/tags | \ +LATEST_VERSION=$(curl -s https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/utils/list-versions/version_date.tsv | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION From 387e0bed4bb7b940b045c5a53926b153ae106dc8 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 16 Mar 2024 19:34:38 +0100 Subject: [PATCH 0147/1165] Fix case when entry might be removed at itarate() by another thread (because of !locked_key) before we call finalize() for it --- src/Interpreters/Cache/EvictionCandidates.cpp | 10 ++-- src/Interpreters/Cache/EvictionCandidates.h | 8 ++-- src/Interpreters/Cache/FileCache.cpp | 10 ++-- src/Interpreters/Cache/FileCache.h | 8 ++-- src/Interpreters/Cache/Guards.h | 22 ++++----- src/Interpreters/Cache/IFileCachePriority.h | 44 ++++++++++++------ .../Cache/LRUFileCachePriority.cpp | 46 +++++++++++-------- src/Interpreters/Cache/LRUFileCachePriority.h | 34 +++++++------- src/Interpreters/Cache/Metadata.cpp | 6 +-- src/Interpreters/Cache/Metadata.h | 25 +++++++++- src/Interpreters/Cache/QueryLimit.cpp | 12 ++--- src/Interpreters/Cache/QueryLimit.h | 12 ++--- .../Cache/SLRUFileCachePriority.cpp | 28 +++++------ .../Cache/SLRUFileCachePriority.h | 24 +++++----- 14 files changed, 167 insertions(+), 122 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 34be84e5407..bfd986fd63f 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -24,22 +24,22 @@ EvictionCandidates::~EvictionCandidates() for (const auto & [key, key_candidates] : candidates) { for (const auto & candidate : key_candidates.candidates) - candidate->removal_candidate = false; + candidate->setEvicting(false, nullptr, nullptr); } } -void EvictionCandidates::add(LockedKey & locked_key, const FileSegmentMetadataPtr & candidate) +void EvictionCandidates::add(const FileSegmentMetadataPtr & candidate, LockedKey & locked_key, const CachePriorityGuard::Lock & lock) { auto [it, inserted] = candidates.emplace(locked_key.getKey(), KeyCandidates{}); if (inserted) it->second.key_metadata = locked_key.getKeyMetadata(); it->second.candidates.push_back(candidate); - candidate->removal_candidate = true; + candidate->setEvicting(true, &locked_key, &lock); ++candidates_size; } -void EvictionCandidates::evict() +void EvictionCandidates::evict(const CachePriorityGuard::Lock &) { if (candidates.empty()) return; @@ -84,7 +84,7 @@ void EvictionCandidates::evict() } } -void EvictionCandidates::finalize(FileCacheQueryLimit::QueryContext * query_context, const CacheGuard::Lock & lock) +void EvictionCandidates::finalize(FileCacheQueryLimit::QueryContext * query_context, const CachePriorityGuard::Lock & lock) { while (!queue_entries_to_invalidate.empty()) { diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index 81681dd7d00..b1f4caedab6 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -9,11 +9,11 @@ class EvictionCandidates public: ~EvictionCandidates(); - void add(LockedKey & locked_key, const FileSegmentMetadataPtr & candidate); + void add(const FileSegmentMetadataPtr & candidate, LockedKey & locked_key, const CachePriorityGuard::Lock &); - void evict(); + void evict(const CachePriorityGuard::Lock &); - void finalize(FileCacheQueryLimit::QueryContext * query_context, const CacheGuard::Lock & lock); + void finalize(FileCacheQueryLimit::QueryContext * query_context, const CachePriorityGuard::Lock &); size_t size() const { return candidates_size; } @@ -21,7 +21,7 @@ public: auto end() const { return candidates.end(); } - using FinalizeEvictionFunc = std::function; + using FinalizeEvictionFunc = std::function; void setFinalizeEvictionFunc(FinalizeEvictionFunc && func) { finalize_eviction_func = func; } private: diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 2daf5e1e1e4..1b65be25fbe 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -182,13 +182,13 @@ void FileCache::initialize() is_initialized = true; } -CacheGuard::Lock FileCache::lockCache() const +CachePriorityGuard::Lock FileCache::lockCache() const { ProfileEventTimeIncrement watch(ProfileEvents::FilesystemCacheLockCacheMicroseconds); return cache_guard.lock(); } -CacheGuard::Lock FileCache::tryLockCache() const +CachePriorityGuard::Lock FileCache::tryLockCache() const { return cache_guard.tryLock(); } @@ -215,7 +215,7 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: return false; FileSegmentPtr file_segment; - if (!file_segment_metadata.evicting()) + if (!file_segment_metadata.isEvicting(locked_key)) { file_segment = file_segment_metadata.file_segment; } @@ -705,7 +705,7 @@ KeyMetadata::iterator FileCache::addFileSegment( size_t size, FileSegment::State state, const CreateFileSegmentSettings & create_settings, - const CacheGuard::Lock * lock) + const CachePriorityGuard::Lock * lock) { /// Create a file_segment_metadata and put it in `files` map by [key][offset]. @@ -844,7 +844,7 @@ bool FileCache::tryReserve( try { /// Remove eviction candidates from filesystem. - eviction_candidates.evict(); + eviction_candidates.evict(cache_lock); } catch (...) { diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 5b665ad0271..655ca285b38 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -172,8 +172,8 @@ public: void deactivateBackgroundOperations(); - CacheGuard::Lock lockCache() const; - CacheGuard::Lock tryLockCache() const; + CachePriorityGuard::Lock lockCache() const; + CachePriorityGuard::Lock tryLockCache() const; std::vector sync(); @@ -207,7 +207,7 @@ private: CacheMetadata metadata; FileCachePriorityPtr main_priority; - mutable CacheGuard cache_guard; + mutable CachePriorityGuard cache_guard; struct HitsCountStash { @@ -279,7 +279,7 @@ private: size_t size, FileSegment::State state, const CreateFileSegmentSettings & create_settings, - const CacheGuard::Lock *); + const CachePriorityGuard::Lock *); }; } diff --git a/src/Interpreters/Cache/Guards.h b/src/Interpreters/Cache/Guards.h index 5729620d82f..1fbf0591cc2 100644 --- a/src/Interpreters/Cache/Guards.h +++ b/src/Interpreters/Cache/Guards.h @@ -10,17 +10,17 @@ namespace DB * 2. KeyGuard::Lock (hold till the end of the method) * * FileCache::tryReserve - * 1. CacheGuard::Lock + * 1. CachePriorityGuard::Lock * 2. KeyGuard::Lock (taken without metadata lock) * 3. any number of KeyGuard::Lock's for files which are going to be evicted (taken via metadata lock) * * FileCache::removeIfExists - * 1. CacheGuard::Lock + * 1. CachePriorityGuard::Lock * 2. KeyGuard::Lock (taken via metadata lock) * 3. FileSegmentGuard::Lock * * FileCache::removeAllReleasable - * 1. CacheGuard::Lock + * 1. CachePriorityGuard::Lock * 2. any number of KeyGuard::Lock's locks (takken via metadata lock), but at a moment of time only one key lock can be hold * 3. FileSegmentGuard::Lock * @@ -34,23 +34,23 @@ namespace DB * 2. FileSegmentGuard::Lock * * FileSegment::complete - * 1. CacheGuard::Lock + * 1. CachePriorityGuard::Lock * 2. KeyGuard::Lock (taken without metadata lock) * 3. FileSegmentGuard::Lock * * Rules: - * 1. Priority of locking: CacheGuard::Lock > CacheMetadataGuard::Lock > KeyGuard::Lock > FileSegmentGuard::Lock - * 2. If we take more than one key lock at a moment of time, we need to take CacheGuard::Lock (example: tryReserve()) + * 1. Priority of locking: CachePriorityGuard::Lock > CacheMetadataGuard::Lock > KeyGuard::Lock > FileSegmentGuard::Lock + * 2. If we take more than one key lock at a moment of time, we need to take CachePriorityGuard::Lock (example: tryReserve()) * * - * _CacheGuard_ + * _CachePriorityGuard_ * 1. FileCache::tryReserve * 2. FileCache::removeIfExists(key) * 3. FileCache::removeAllReleasable * 4. FileSegment::complete * * _KeyGuard_ _CacheMetadataGuard_ - * 1. all from CacheGuard 1. getOrSet/get/set + * 1. all from CachePriorityGuard 1. getOrSet/get/set * 2. getOrSet/get/Set * * *This table does not include locks taken for introspection and system tables. @@ -59,10 +59,10 @@ namespace DB /** * Cache priority queue guard. */ -struct CacheGuard : private boost::noncopyable +struct CachePriorityGuard : private boost::noncopyable { - /// struct is used (not keyword `using`) to make CacheGuard::Lock non-interchangable with other guards locks - /// so, we wouldn't be able to pass CacheGuard::Lock to a function which accepts KeyGuard::Lock, for example + /// struct is used (not keyword `using`) to make CachePriorityGuard::Lock non-interchangable with other guards locks + /// so, we wouldn't be able to pass CachePriorityGuard::Lock to a function which accepts KeyGuard::Lock, for example struct Lock : public std::unique_lock { using Base = std::unique_lock; diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index 37270cf0873..651710656af 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -14,6 +14,11 @@ namespace DB struct FileCacheReserveStat; class EvictionCandidates; +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + class IFileCachePriority : private boost::noncopyable { public: @@ -33,6 +38,19 @@ public: std::atomic size; size_t hits = 0; + + bool isEvicting(const CachePriorityGuard::Lock &) const { return evicting; } + bool isEvicting(const LockedKey &) const { return evicting; } + void setEvicting(bool evicting_, const LockedKey * locked_key, const CachePriorityGuard::Lock * lock) const + { + if (evicting_ && (!locked_key || !lock)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Setting evicting state to `true` can be done only under lock"); + evicting.store(evicting_); + } + + private: + mutable std::atomic evicting = false; }; using EntryPtr = std::shared_ptr; @@ -43,16 +61,16 @@ public: virtual EntryPtr getEntry() const = 0; - virtual size_t increasePriority(const CacheGuard::Lock &) = 0; + virtual size_t increasePriority(const CachePriorityGuard::Lock &) = 0; /// Note: IncrementSize unlike decrementSize requires a cache lock, because /// it requires more consistency guarantees for eviction. - virtual void incrementSize(size_t size, const CacheGuard::Lock &) = 0; + virtual void incrementSize(size_t size, const CachePriorityGuard::Lock &) = 0; virtual void decrementSize(size_t size) = 0; - virtual void remove(const CacheGuard::Lock &) = 0; + virtual void remove(const CachePriorityGuard::Lock &) = 0; virtual void invalidate() = 0; @@ -62,13 +80,13 @@ public: virtual ~IFileCachePriority() = default; - size_t getElementsLimit(const CacheGuard::Lock &) const { return max_elements; } + size_t getElementsLimit(const CachePriorityGuard::Lock &) const { return max_elements; } - size_t getSizeLimit(const CacheGuard::Lock &) const { return max_size; } + size_t getSizeLimit(const CachePriorityGuard::Lock &) const { return max_size; } - virtual size_t getSize(const CacheGuard::Lock &) const = 0; + virtual size_t getSize(const CachePriorityGuard::Lock &) const = 0; - virtual size_t getElementsCount(const CacheGuard::Lock &) const = 0; + virtual size_t getElementsCount(const CachePriorityGuard::Lock &) const = 0; /// Throws exception if there is not enough size to fit it. virtual IteratorPtr add( /// NOLINT @@ -76,7 +94,7 @@ public: size_t offset, size_t size, const UserInfo & user, - const CacheGuard::Lock &, + const CachePriorityGuard::Lock &, bool best_effort = false) = 0; /// `reservee` is the entry for which are reserving now. @@ -84,11 +102,11 @@ public: /// for the corresponding file segment. virtual bool canFit( /// NOLINT size_t size, - const CacheGuard::Lock &, + const CachePriorityGuard::Lock &, IteratorPtr reservee = nullptr, bool best_effort = false) const = 0; - virtual void shuffle(const CacheGuard::Lock &) = 0; + virtual void shuffle(const CachePriorityGuard::Lock &) = 0; struct IPriorityDump { @@ -96,7 +114,7 @@ public: }; using PriorityDumpPtr = std::shared_ptr; - virtual PriorityDumpPtr dump(const CacheGuard::Lock &) = 0; + virtual PriorityDumpPtr dump(const CachePriorityGuard::Lock &) = 0; virtual bool collectCandidatesForEviction( size_t size, @@ -104,9 +122,9 @@ public: EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, const UserID & user_id, - const CacheGuard::Lock &) = 0; + const CachePriorityGuard::Lock &) = 0; - virtual bool modifySizeLimits(size_t max_size_, size_t max_elements_, double size_ratio_, const CacheGuard::Lock &) = 0; + virtual bool modifySizeLimits(size_t max_size_, size_t max_elements_, double size_ratio_, const CachePriorityGuard::Lock &) = 0; protected: IFileCachePriority(size_t max_size_, size_t max_elements_); diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 0f164f25293..65138fd70fc 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -44,13 +44,13 @@ IFileCachePriority::IteratorPtr LRUFileCachePriority::add( /// NOLINT size_t offset, size_t size, const UserInfo &, - const CacheGuard::Lock & lock, + const CachePriorityGuard::Lock & lock, bool) { return std::make_shared(add(std::make_shared(key_metadata->key, offset, size, key_metadata), lock)); } -LRUFileCachePriority::LRUIterator LRUFileCachePriority::add(EntryPtr entry, const CacheGuard::Lock & lock) +LRUFileCachePriority::LRUIterator LRUFileCachePriority::add(EntryPtr entry, const CachePriorityGuard::Lock & lock) { if (entry->size == 0) { @@ -94,7 +94,7 @@ LRUFileCachePriority::LRUIterator LRUFileCachePriority::add(EntryPtr entry, cons return LRUIterator(this, iterator); } -LRUFileCachePriority::LRUQueue::iterator LRUFileCachePriority::remove(LRUQueue::iterator it, const CacheGuard::Lock &) +LRUFileCachePriority::LRUQueue::iterator LRUFileCachePriority::remove(LRUQueue::iterator it, const CachePriorityGuard::Lock &) { /// If size is 0, entry is invalidated, current_elements_num was already updated. const auto & entry = **it; @@ -154,11 +154,21 @@ bool LRUFileCachePriority::LRUIterator::operator ==(const LRUIterator & other) c return cache_priority == other.cache_priority && iterator == other.iterator; } -void LRUFileCachePriority::iterate(IterateFunc && func, const CacheGuard::Lock & lock) +void LRUFileCachePriority::iterate(IterateFunc && func, const CachePriorityGuard::Lock & lock) { for (auto it = queue.begin(); it != queue.end();) { const auto & entry = **it; + + if (entry.size == 0) + { + it = remove(it, lock); + continue; + } + + if (entry.isEvicting(lock)) + continue; + auto locked_key = entry.key_metadata->tryLock(); if (!locked_key || entry.size == 0) { @@ -205,7 +215,7 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CacheGuard::Lock & bool LRUFileCachePriority::canFit( /// NOLINT size_t size, - const CacheGuard::Lock & lock, + const CachePriorityGuard::Lock & lock, IteratorPtr, bool) const { @@ -216,7 +226,7 @@ bool LRUFileCachePriority::canFit( size_t size, size_t released_size_assumption, size_t released_elements_assumption, - const CacheGuard::Lock &) const + const CachePriorityGuard::Lock &) const { return (max_size == 0 || (state->current_size + size - released_size_assumption <= max_size)) && (max_elements == 0 || state->current_elements_num + 1 - released_elements_assumption <= max_elements); @@ -228,7 +238,7 @@ bool LRUFileCachePriority::collectCandidatesForEviction( EvictionCandidates & res, IFileCachePriority::IteratorPtr, const UserID &, - const CacheGuard::Lock & lock) + const CachePriorityGuard::Lock & lock) { if (canFit(size, lock)) return true; @@ -240,13 +250,9 @@ bool LRUFileCachePriority::collectCandidatesForEviction( const auto & file_segment = segment_metadata->file_segment; chassert(file_segment->assertCorrectness()); - if (segment_metadata->evicting()) + if (segment_metadata->releasable()) { - ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictionSkippedEvictingFileSegments); - } - else if (segment_metadata->releasable()) - { - res.add(locked_key, segment_metadata); + res.add(segment_metadata, locked_key, lock); stat.update(segment_metadata->size(), file_segment->getKind(), true); } else @@ -271,7 +277,7 @@ bool LRUFileCachePriority::collectCandidatesForEviction( return can_fit(); } -LRUFileCachePriority::LRUIterator LRUFileCachePriority::move(LRUIterator & it, LRUFileCachePriority & other, const CacheGuard::Lock &) +LRUFileCachePriority::LRUIterator LRUFileCachePriority::move(LRUIterator & it, LRUFileCachePriority & other, const CachePriorityGuard::Lock &) { const auto & entry = *it.getEntry(); if (entry.size == 0) @@ -304,7 +310,7 @@ LRUFileCachePriority::LRUIterator LRUFileCachePriority::move(LRUIterator & it, L return LRUIterator(this, it.iterator); } -IFileCachePriority::PriorityDumpPtr LRUFileCachePriority::dump(const CacheGuard::Lock & lock) +IFileCachePriority::PriorityDumpPtr LRUFileCachePriority::dump(const CachePriorityGuard::Lock & lock) { std::vector res; iterate([&](LockedKey &, const FileSegmentMetadataPtr & segment_metadata) @@ -316,7 +322,7 @@ IFileCachePriority::PriorityDumpPtr LRUFileCachePriority::dump(const CacheGuard: } bool LRUFileCachePriority::modifySizeLimits( - size_t max_size_, size_t max_elements_, double /* size_ratio_ */, const CacheGuard::Lock & lock) + size_t max_size_, size_t max_elements_, double /* size_ratio_ */, const CachePriorityGuard::Lock & lock) { if (max_size == max_size_ && max_elements == max_elements_) return false; /// Nothing to change. @@ -360,7 +366,7 @@ bool LRUFileCachePriority::modifySizeLimits( return true; } -void LRUFileCachePriority::LRUIterator::remove(const CacheGuard::Lock & lock) +void LRUFileCachePriority::LRUIterator::remove(const CachePriorityGuard::Lock & lock) { assertValid(); cache_priority->remove(iterator, lock); @@ -383,7 +389,7 @@ void LRUFileCachePriority::LRUIterator::invalidate() entry->size = 0; } -void LRUFileCachePriority::LRUIterator::incrementSize(size_t size, const CacheGuard::Lock &) +void LRUFileCachePriority::LRUIterator::incrementSize(size_t size, const CachePriorityGuard::Lock &) { assertValid(); @@ -415,7 +421,7 @@ void LRUFileCachePriority::LRUIterator::decrementSize(size_t size) entry->size -= size; } -size_t LRUFileCachePriority::LRUIterator::increasePriority(const CacheGuard::Lock &) +size_t LRUFileCachePriority::LRUIterator::increasePriority(const CachePriorityGuard::Lock &) { assertValid(); cache_priority->queue.splice(cache_priority->queue.end(), cache_priority->queue, iterator); @@ -428,7 +434,7 @@ void LRUFileCachePriority::LRUIterator::assertValid() const throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to use invalid iterator"); } -void LRUFileCachePriority::shuffle(const CacheGuard::Lock &) +void LRUFileCachePriority::shuffle(const CachePriorityGuard::Lock &) { std::vector its; its.reserve(queue.size()); diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index d8907f678a2..d5151b6bbff 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -24,13 +24,13 @@ protected: public: LRUFileCachePriority(size_t max_size_, size_t max_elements_, StatePtr state_ = nullptr); - size_t getSize(const CacheGuard::Lock &) const override { return state->current_size; } + size_t getSize(const CachePriorityGuard::Lock &) const override { return state->current_size; } - size_t getElementsCount(const CacheGuard::Lock &) const override { return state->current_elements_num; } + size_t getElementsCount(const CachePriorityGuard::Lock &) const override { return state->current_elements_num; } bool canFit( /// NOLINT size_t size, - const CacheGuard::Lock &, + const CachePriorityGuard::Lock &, IteratorPtr reservee = nullptr, bool best_effort = false) const override; @@ -39,7 +39,7 @@ public: size_t offset, size_t size, const UserInfo & user, - const CacheGuard::Lock &, + const CachePriorityGuard::Lock &, bool best_effort = false) override; bool collectCandidatesForEviction( @@ -48,9 +48,9 @@ public: EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, const UserID & user_id, - const CacheGuard::Lock &) override; + const CachePriorityGuard::Lock &) override; - void shuffle(const CacheGuard::Lock &) override; + void shuffle(const CachePriorityGuard::Lock &) override; struct LRUPriorityDump : public IPriorityDump { @@ -58,11 +58,11 @@ public: explicit LRUPriorityDump(const std::vector & infos_) : infos(infos_) {} void merge(const LRUPriorityDump & other) { infos.insert(infos.end(), other.infos.begin(), other.infos.end()); } }; - PriorityDumpPtr dump(const CacheGuard::Lock &) override; + PriorityDumpPtr dump(const CachePriorityGuard::Lock &) override; - void pop(const CacheGuard::Lock & lock) { remove(queue.begin(), lock); } + void pop(const CachePriorityGuard::Lock & lock) { remove(queue.begin(), lock); } - bool modifySizeLimits(size_t max_size_, size_t max_elements_, double size_ratio_, const CacheGuard::Lock &) override; + bool modifySizeLimits(size_t max_size_, size_t max_elements_, double size_ratio_, const CachePriorityGuard::Lock &) override; private: class LRUIterator; @@ -76,9 +76,9 @@ private: void updateElementsCount(int64_t num); void updateSize(int64_t size); - bool canFit(size_t size, size_t released_size_assumption, size_t released_elements_assumption, const CacheGuard::Lock &) const; + bool canFit(size_t size, size_t released_size_assumption, size_t released_elements_assumption, const CachePriorityGuard::Lock &) const; - LRUQueue::iterator remove(LRUQueue::iterator it, const CacheGuard::Lock &); + LRUQueue::iterator remove(LRUQueue::iterator it, const CachePriorityGuard::Lock &); enum class IterationResult { @@ -87,10 +87,10 @@ private: REMOVE_AND_CONTINUE, }; using IterateFunc = std::function; - void iterate(IterateFunc && func, const CacheGuard::Lock &); + void iterate(IterateFunc && func, const CachePriorityGuard::Lock &); - LRUIterator move(LRUIterator & it, LRUFileCachePriority & other, const CacheGuard::Lock &); - LRUIterator add(EntryPtr entry, const CacheGuard::Lock &); + LRUIterator move(LRUIterator & it, LRUFileCachePriority & other, const CachePriorityGuard::Lock &); + LRUIterator add(EntryPtr entry, const CachePriorityGuard::Lock &); }; class LRUFileCachePriority::LRUIterator : public IFileCachePriority::Iterator @@ -107,13 +107,13 @@ public: EntryPtr getEntry() const override { return *iterator; } - size_t increasePriority(const CacheGuard::Lock &) override; + size_t increasePriority(const CachePriorityGuard::Lock &) override; - void remove(const CacheGuard::Lock &) override; + void remove(const CachePriorityGuard::Lock &) override; void invalidate() override; - void incrementSize(size_t size, const CacheGuard::Lock &) override; + void incrementSize(size_t size, const CachePriorityGuard::Lock &) override; void decrementSize(size_t size) override; diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index d3fb699a115..da4ae5f29d5 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -614,7 +614,7 @@ void CacheMetadata::downloadThreadFunc(const bool & stop_flag) continue; auto file_segment_metadata = locked_key->tryGetByOffset(offset); - if (!file_segment_metadata || file_segment_metadata->evicting()) + if (!file_segment_metadata || file_segment_metadata->isEvicting(*locked_key)) continue; auto file_segment = file_segment_weak.lock(); @@ -877,7 +877,7 @@ bool LockedKey::removeAllFileSegments(bool if_releasable) removed_all = false; continue; } - else if (it->second->evicting()) + else if (it->second->isEvicting(*this)) { /// File segment is currently a removal candidate, /// we do not know if it will be removed or not yet, @@ -1100,7 +1100,7 @@ std::vector LockedKey::sync() std::vector broken; for (auto it = key_metadata->begin(); it != key_metadata->end();) { - if (it->second->evicting() || !it->second->releasable()) + if (it->second->isEvicting(*this) || !it->second->releasable()) { ++it; continue; diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index adcc1545307..7c08bcbe80e 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -30,12 +30,33 @@ struct FileSegmentMetadata : private boost::noncopyable size_t size() const; - bool evicting() const { return removal_candidate.load(); } + bool isEvicting(const CachePriorityGuard::Lock & lock) const + { + auto iterator = getQueueIterator(); + if (!iterator) + return false; + return iterator->getEntry()->isEvicting(lock); + } + + bool isEvicting(const LockedKey & lock) const + { + auto iterator = getQueueIterator(); + if (!iterator) + return false; + return iterator->getEntry()->isEvicting(lock); + } + + void setEvicting(bool evicting, const LockedKey * locked_key, const CachePriorityGuard::Lock * lock) const + { + auto iterator = getQueueIterator(); + if (!iterator) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Iterator is not set"); + iterator->getEntry()->setEvicting(evicting, locked_key, lock); + } Priority::IteratorPtr getQueueIterator() const { return file_segment->getQueueIterator(); } FileSegmentPtr file_segment; - std::atomic removal_candidate{false}; }; using FileSegmentMetadataPtr = std::shared_ptr; diff --git a/src/Interpreters/Cache/QueryLimit.cpp b/src/Interpreters/Cache/QueryLimit.cpp index ba4f5017772..9421005dc92 100644 --- a/src/Interpreters/Cache/QueryLimit.cpp +++ b/src/Interpreters/Cache/QueryLimit.cpp @@ -16,7 +16,7 @@ static bool isQueryInitialized() && !CurrentThread::getQueryId().empty(); } -FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::tryGetQueryContext(const CacheGuard::Lock &) +FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::tryGetQueryContext(const CachePriorityGuard::Lock &) { if (!isQueryInitialized()) return nullptr; @@ -25,7 +25,7 @@ FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::tryGetQueryContext(con return (query_iter == query_map.end()) ? nullptr : query_iter->second; } -void FileCacheQueryLimit::removeQueryContext(const std::string & query_id, const CacheGuard::Lock &) +void FileCacheQueryLimit::removeQueryContext(const std::string & query_id, const CachePriorityGuard::Lock &) { auto query_iter = query_map.find(query_id); if (query_iter == query_map.end()) @@ -41,7 +41,7 @@ void FileCacheQueryLimit::removeQueryContext(const std::string & query_id, const FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::getOrSetQueryContext( const std::string & query_id, const ReadSettings & settings, - const CacheGuard::Lock &) + const CachePriorityGuard::Lock &) { if (query_id.empty()) return nullptr; @@ -70,7 +70,7 @@ void FileCacheQueryLimit::QueryContext::add( size_t offset, size_t size, const FileCache::UserInfo & user, - const CacheGuard::Lock & lock) + const CachePriorityGuard::Lock & lock) { auto it = getPriority().add(key_metadata, offset, size, user, lock); auto [_, inserted] = records.emplace(FileCacheKeyAndOffset{key_metadata->key, offset}, it); @@ -87,7 +87,7 @@ void FileCacheQueryLimit::QueryContext::add( void FileCacheQueryLimit::QueryContext::remove( const Key & key, size_t offset, - const CacheGuard::Lock & lock) + const CachePriorityGuard::Lock & lock) { auto record = records.find({key, offset}); if (record == records.end()) @@ -100,7 +100,7 @@ void FileCacheQueryLimit::QueryContext::remove( IFileCachePriority::IteratorPtr FileCacheQueryLimit::QueryContext::tryGet( const Key & key, size_t offset, - const CacheGuard::Lock &) + const CachePriorityGuard::Lock &) { auto it = records.find({key, offset}); if (it == records.end()) diff --git a/src/Interpreters/Cache/QueryLimit.h b/src/Interpreters/Cache/QueryLimit.h index 419126601f0..7553eff82ba 100644 --- a/src/Interpreters/Cache/QueryLimit.h +++ b/src/Interpreters/Cache/QueryLimit.h @@ -13,14 +13,14 @@ public: class QueryContext; using QueryContextPtr = std::shared_ptr; - QueryContextPtr tryGetQueryContext(const CacheGuard::Lock & lock); + QueryContextPtr tryGetQueryContext(const CachePriorityGuard::Lock & lock); QueryContextPtr getOrSetQueryContext( const std::string & query_id, const ReadSettings & settings, - const CacheGuard::Lock &); + const CachePriorityGuard::Lock &); - void removeQueryContext(const std::string & query_id, const CacheGuard::Lock &); + void removeQueryContext(const std::string & query_id, const CachePriorityGuard::Lock &); class QueryContext { @@ -38,19 +38,19 @@ public: Priority::IteratorPtr tryGet( const Key & key, size_t offset, - const CacheGuard::Lock &); + const CachePriorityGuard::Lock &); void add( KeyMetadataPtr key_metadata, size_t offset, size_t size, const FileCacheUserInfo & user, - const CacheGuard::Lock &); + const CachePriorityGuard::Lock &); void remove( const Key & key, size_t offset, - const CacheGuard::Lock &); + const CachePriorityGuard::Lock &); private: using Records = std::unordered_map; diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 84133bc5c56..aca59ede6c2 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -34,19 +34,19 @@ SLRUFileCachePriority::SLRUFileCachePriority( probationary_queue.max_size, protected_queue.max_elements); } -size_t SLRUFileCachePriority::getSize(const CacheGuard::Lock & lock) const +size_t SLRUFileCachePriority::getSize(const CachePriorityGuard::Lock & lock) const { return protected_queue.getSize(lock) + probationary_queue.getSize(lock); } -size_t SLRUFileCachePriority::getElementsCount(const CacheGuard::Lock & lock) const +size_t SLRUFileCachePriority::getElementsCount(const CachePriorityGuard::Lock & lock) const { return protected_queue.getElementsCount(lock) + probationary_queue.getElementsCount(lock); } bool SLRUFileCachePriority::canFit( /// NOLINT size_t size, - const CacheGuard::Lock & lock, + const CachePriorityGuard::Lock & lock, IteratorPtr reservee, bool best_effort) const { @@ -70,7 +70,7 @@ IFileCachePriority::IteratorPtr SLRUFileCachePriority::add( /// NOLINT size_t offset, size_t size, const UserInfo &, - const CacheGuard::Lock & lock, + const CachePriorityGuard::Lock & lock, bool is_startup) { if (is_startup) @@ -102,7 +102,7 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, const UserID & user_id, - const CacheGuard::Lock & lock) + const CachePriorityGuard::Lock & lock) { /// If `it` is nullptr, then it is the first space reservation attempt /// for a corresponding file segment, so it will be directly put into probationary queue. @@ -141,7 +141,7 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( && !probationary_queue.collectCandidatesForEviction(size_to_downgrade, stat, res, reservee, user_id, lock)) return false; - res.setFinalizeEvictionFunc([=, this](const CacheGuard::Lock & lk) mutable + res.setFinalizeEvictionFunc([=, this](const CachePriorityGuard::Lock & lk) mutable { for (const auto & [key, key_candidates] : *downgrade_candidates) { @@ -157,7 +157,7 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( return true; } -void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const CacheGuard::Lock & lock) +void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const CachePriorityGuard::Lock & lock) { /// If entry is already in protected queue, /// we only need to increase its priority within the protected queue. @@ -217,7 +217,7 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach return; } /// Make space for "downgrade" candidates. - eviction_candidates.evict(); + eviction_candidates.evict(lock); eviction_candidates.finalize(nullptr, lock); } @@ -240,7 +240,7 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach iterator.is_protected = true; } -IFileCachePriority::PriorityDumpPtr SLRUFileCachePriority::dump(const CacheGuard::Lock & lock) +IFileCachePriority::PriorityDumpPtr SLRUFileCachePriority::dump(const CachePriorityGuard::Lock & lock) { auto res = dynamic_pointer_cast(probationary_queue.dump(lock)); auto part_res = dynamic_pointer_cast(protected_queue.dump(lock)); @@ -248,14 +248,14 @@ IFileCachePriority::PriorityDumpPtr SLRUFileCachePriority::dump(const CacheGuard return res; } -void SLRUFileCachePriority::shuffle(const CacheGuard::Lock & lock) +void SLRUFileCachePriority::shuffle(const CachePriorityGuard::Lock & lock) { protected_queue.shuffle(lock); probationary_queue.shuffle(lock); } bool SLRUFileCachePriority::modifySizeLimits( - size_t max_size_, size_t max_elements_, double size_ratio_, const CacheGuard::Lock & lock) + size_t max_size_, size_t max_elements_, double size_ratio_, const CachePriorityGuard::Lock & lock) { if (max_size == max_size_ && max_elements == max_elements_ && size_ratio == size_ratio_) return false; /// Nothing to change. @@ -285,14 +285,14 @@ SLRUFileCachePriority::EntryPtr SLRUFileCachePriority::SLRUIterator::getEntry() return entry; } -size_t SLRUFileCachePriority::SLRUIterator::increasePriority(const CacheGuard::Lock & lock) +size_t SLRUFileCachePriority::SLRUIterator::increasePriority(const CachePriorityGuard::Lock & lock) { assertValid(); cache_priority->increasePriority(*this, lock); return getEntry()->hits; } -void SLRUFileCachePriority::SLRUIterator::incrementSize(size_t size, const CacheGuard::Lock & lock) +void SLRUFileCachePriority::SLRUIterator::incrementSize(size_t size, const CachePriorityGuard::Lock & lock) { assertValid(); lru_iterator.incrementSize(size, lock); @@ -310,7 +310,7 @@ void SLRUFileCachePriority::SLRUIterator::invalidate() lru_iterator.invalidate(); } -void SLRUFileCachePriority::SLRUIterator::remove(const CacheGuard::Lock & lock) +void SLRUFileCachePriority::SLRUIterator::remove(const CachePriorityGuard::Lock & lock) { assertValid(); lru_iterator.remove(lock); diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h index 28e61396572..7b8ae77cc8f 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.h +++ b/src/Interpreters/Cache/SLRUFileCachePriority.h @@ -21,13 +21,13 @@ public: LRUFileCachePriority::StatePtr probationary_state_ = nullptr, LRUFileCachePriority::StatePtr protected_state_ = nullptr); - size_t getSize(const CacheGuard::Lock & lock) const override; + size_t getSize(const CachePriorityGuard::Lock & lock) const override; - size_t getElementsCount(const CacheGuard::Lock &) const override; + size_t getElementsCount(const CachePriorityGuard::Lock &) const override; bool canFit( /// NOLINT size_t size, - const CacheGuard::Lock &, + const CachePriorityGuard::Lock &, IteratorPtr reservee = nullptr, bool best_effort = false) const override; @@ -36,7 +36,7 @@ public: size_t offset, size_t size, const UserInfo & user, - const CacheGuard::Lock &, + const CachePriorityGuard::Lock &, bool is_startup = false) override; bool collectCandidatesForEviction( @@ -45,13 +45,13 @@ public: EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, const UserID & user_id, - const CacheGuard::Lock &) override; + const CachePriorityGuard::Lock &) override; - void shuffle(const CacheGuard::Lock &) override; + void shuffle(const CachePriorityGuard::Lock &) override; - PriorityDumpPtr dump(const CacheGuard::Lock &) override; + PriorityDumpPtr dump(const CachePriorityGuard::Lock &) override; - bool modifySizeLimits(size_t max_size_, size_t max_elements_, double size_ratio_, const CacheGuard::Lock &) override; + bool modifySizeLimits(size_t max_size_, size_t max_elements_, double size_ratio_, const CachePriorityGuard::Lock &) override; private: double size_ratio; @@ -59,7 +59,7 @@ private: LRUFileCachePriority probationary_queue; LoggerPtr log = getLogger("SLRUFileCachePriority"); - void increasePriority(SLRUIterator & iterator, const CacheGuard::Lock & lock); + void increasePriority(SLRUIterator & iterator, const CachePriorityGuard::Lock & lock); }; class SLRUFileCachePriority::SLRUIterator : public IFileCachePriority::Iterator @@ -73,13 +73,13 @@ public: EntryPtr getEntry() const override; - size_t increasePriority(const CacheGuard::Lock &) override; + size_t increasePriority(const CachePriorityGuard::Lock &) override; - void remove(const CacheGuard::Lock &) override; + void remove(const CachePriorityGuard::Lock &) override; void invalidate() override; - void incrementSize(size_t size, const CacheGuard::Lock &) override; + void incrementSize(size_t size, const CachePriorityGuard::Lock &) override; void decrementSize(size_t size) override; From 41d8621d975497f7a5b0c1fbe845a6410a2a5341 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 14 Mar 2024 23:50:01 +0100 Subject: [PATCH 0148/1165] Fix replacing dictionaries: StorageID is now updated immediately for every dictionary during CREATE OR REPLACE even if the dictionary is not loaded yet or loading right now. --- src/Dictionaries/IDictionary.h | 18 +++++++------ .../ExternalDictionariesLoader.cpp | 14 ++++++++++ src/Interpreters/ExternalDictionariesLoader.h | 3 +++ src/Interpreters/ExternalLoader.cpp | 21 ++++++++++++--- src/Interpreters/ExternalLoader.h | 4 +++ src/Storages/StorageDictionary.cpp | 27 ++++++++++--------- 6 files changed, 63 insertions(+), 24 deletions(-) diff --git a/src/Dictionaries/IDictionary.h b/src/Dictionaries/IDictionary.h index bab3e470b79..9770e93669e 100644 --- a/src/Dictionaries/IDictionary.h +++ b/src/Dictionaries/IDictionary.h @@ -75,13 +75,6 @@ public: return dictionary_id; } - void updateDictionaryName(const StorageID & new_name) const - { - std::lock_guard lock{mutex}; - assert(new_name.uuid == dictionary_id.uuid && dictionary_id.uuid != UUIDHelpers::Nil); - dictionary_id = new_name; - } - std::string getLoadableName() const final { std::lock_guard lock{mutex}; @@ -452,9 +445,18 @@ public: return sample_block; } + /// Internally called by ExternalDictionariesLoader. + /// In order to update the dictionary ID change its configuration first and then call ExternalDictionariesLoader::reloadConfig(). + void updateDictionaryID(const StorageID & new_dictionary_id) + { + std::lock_guard lock{mutex}; + assert((new_dictionary_id.uuid == dictionary_id.uuid) && (dictionary_id.uuid != UUIDHelpers::Nil)); + dictionary_id = new_dictionary_id; + } + private: mutable std::mutex mutex; - mutable StorageID dictionary_id TSA_GUARDED_BY(mutex); + StorageID dictionary_id TSA_GUARDED_BY(mutex); String dictionary_comment TSA_GUARDED_BY(mutex); }; diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 74984de0064..760f94d18c2 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -41,6 +41,20 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( return DictionaryFactory::instance().create(name, config, key_in_config, getContext(), created_from_ddl); } +void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const +{ + IDictionary & dict = static_cast(object); + + auto new_dictionary_id = StorageID::fromDictionaryConfig(config, key_in_config); + auto old_dictionary_id = dict.getDictionaryID(); + if ((new_dictionary_id.table_name != old_dictionary_id.table_name) || (new_dictionary_id.database_name != old_dictionary_id.database_name)) + { + /// We can update the dictionary ID without reloading only if it's in the atomic database. + if ((new_dictionary_id.uuid == old_dictionary_id.uuid) && (new_dictionary_id.uuid != UUIDHelpers::Nil)) + dict.updateDictionaryID(new_dictionary_id); + } +} + ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::getDictionary(const std::string & dictionary_name, ContextPtr local_context) const { std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h index 5f27d695c8b..3c226f998be 100644 --- a/src/Interpreters/ExternalDictionariesLoader.h +++ b/src/Interpreters/ExternalDictionariesLoader.h @@ -43,6 +43,9 @@ protected: LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string & repository_name) const override; + void updateObjectFromConfigWithoutReloading( + IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const override; + std::string resolveDictionaryName(const std::string & dictionary_name, const std::string & current_database_name) const; /// Try convert qualified dictionary name to persistent UUID diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 36664cbd06f..ec429054475 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -87,6 +87,8 @@ namespace lock = std::unique_lock(mutex); } }; + + using UpdateObjectFromConfigWithoutReloadingFunction = std::function; } @@ -377,7 +379,7 @@ private: } const String type_name; - LoggerPtr log; + const LoggerPtr log; std::mutex mutex; ExternalLoaderConfigSettings settings; @@ -400,9 +402,11 @@ public: LoadingDispatcher( const CreateObjectFunction & create_object_function_, + const UpdateObjectFromConfigWithoutReloadingFunction & update_object_from_config_without_reloading_, const String & type_name_, LoggerPtr log_) : create_object(create_object_function_) + , update_object_from_config_without_reloading(update_object_from_config_without_reloading_) , type_name(type_name_) , log(log_) { @@ -457,10 +461,15 @@ public: else { const auto & new_config = new_config_it->second; - bool config_is_same = isSameConfiguration(*info.config->config, info.config->key_in_config, *new_config->config, new_config->key_in_config); + auto previous_config = info.config; info.config = new_config; - if (!config_is_same) + + bool config_changed = !isSameConfiguration(*previous_config->config, previous_config->key_in_config, *new_config->config, new_config->key_in_config); + if (config_changed) { + if (info.object) + update_object_from_config_without_reloading(*info.object, *new_config->config, new_config->key_in_config); + if (info.triedToLoad()) { /// The object has been tried to load before, so it is currently in use or was in use @@ -1117,7 +1126,10 @@ private: } if (new_object) + { + update_object_from_config_without_reloading(*new_object, *info->config->config, info->config->key_in_config); info->object = new_object; + } info->exception = new_exception; info->error_count = error_count; @@ -1192,6 +1204,7 @@ private: } const CreateObjectFunction create_object; + const UpdateObjectFromConfigWithoutReloadingFunction update_object_from_config_without_reloading; const String type_name; LoggerPtr log; @@ -1277,6 +1290,8 @@ ExternalLoader::ExternalLoader(const String & type_name_, LoggerPtr log_) : config_files_reader(std::make_unique(type_name_, log_)) , loading_dispatcher(std::make_unique( [this](auto && a, auto && b, auto && c) { return createObject(a, b, c); }, + [this](const IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) + { return updateObjectFromConfigWithoutReloading(const_cast(object), config, key_in_config); }, type_name_, log_)) , periodic_updater(std::make_unique(*config_files_reader, *loading_dispatcher)) diff --git a/src/Interpreters/ExternalLoader.h b/src/Interpreters/ExternalLoader.h index a5d83bdab50..279157b6d51 100644 --- a/src/Interpreters/ExternalLoader.h +++ b/src/Interpreters/ExternalLoader.h @@ -213,6 +213,10 @@ public: protected: virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0; + /// Updates the object from the configuration without reloading as much as possible. + virtual void updateObjectFromConfigWithoutReloading( + IExternalLoadable & /* object */, const Poco::Util::AbstractConfiguration & /* config */, const String & /* key_in_config */) const {} + private: void checkLoaded(const LoadResult & result, bool check_no_errors) const; void checkLoaded(const LoadResults & results, bool check_no_errors) const; diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index f716332dd24..0c6f6a25b9d 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -221,6 +221,10 @@ void StorageDictionary::renameInMemory(const StorageID & new_table_id) bool move_to_ordinary = old_table_id.uuid != UUIDHelpers::Nil && new_table_id.uuid == UUIDHelpers::Nil; assert(old_table_id.uuid == new_table_id.uuid || move_to_atomic || move_to_ordinary); + /// It's better not to update an associated `IDictionary` directly here because it can be not loaded yet or + /// it can be in the process of loading or reloading right now. + /// The correct way is to update the dictionary's configuration first and then ask ExternalDictionariesLoader to reload our dictionary. + { std::lock_guard lock(dictionary_config_mutex); @@ -232,31 +236,28 @@ void StorageDictionary::renameInMemory(const StorageID & new_table_id) configuration->remove("dictionary.uuid"); } + const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); + /// Dictionary is moving between databases of different engines or is renaming inside Ordinary database bool recreate_dictionary = old_table_id.uuid == UUIDHelpers::Nil || new_table_id.uuid == UUIDHelpers::Nil; if (recreate_dictionary) { - /// It's too hard to update both name and uuid, better to reload dictionary with new name + /// For an ordinary database the config repositories of dictionaries are identified by the full name (database name + dictionary name), + /// so we cannot change the dictionary name or the database name on the fly (without extra reloading) and have to recreate the config repository. removeDictionaryConfigurationFromRepository(); auto repository = std::make_unique(*this); - remove_repository_callback = getContext()->getExternalDictionariesLoader().addConfigRepository(std::move(repository)); - /// Dictionary will be reloaded lazily to avoid exceptions in the middle of renaming + remove_repository_callback = external_dictionaries_loader.addConfigRepository(std::move(repository)); + /// Dictionary will be now reloaded lazily. } else { - const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); - auto result = external_dictionaries_loader.getLoadResult(old_table_id.getInternalDictionaryName()); - - if (result.object) - { - const auto dictionary = std::static_pointer_cast(result.object); - dictionary->updateDictionaryName(new_table_id); - } - + /// For an atomic database dictionaries are identified inside the ExternalLoader by UUID, + /// so we can change the dictionary name or the database name on the fly (without extra reloading) because UUID doesn't change. external_dictionaries_loader.reloadConfig(old_table_id.getInternalDictionaryName()); - dictionary_name = new_table_id.getFullNameNotQuoted(); } + + dictionary_name = new_table_id.getFullNameNotQuoted(); } void StorageDictionary::checkAlterIsPossible(const AlterCommands & commands, ContextPtr /* context */) const From 4fe1c9a78f99b915dd4f31cbf37a2c3db6f49e1d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 15 Mar 2024 00:07:53 +0100 Subject: [PATCH 0149/1165] Get rid of checking the "last modification time" when the configuration is generated to allow ExternalLoader::reloadConfig() work for generated configurations too. --- src/Interpreters/ExternalLoader.cpp | 4 ++-- .../ExternalLoaderDictionaryStorageConfigRepository.cpp | 5 ----- .../ExternalLoaderDictionaryStorageConfigRepository.h | 2 -- src/Interpreters/ExternalLoaderTempConfigRepository.cpp | 8 -------- src/Interpreters/ExternalLoaderTempConfigRepository.h | 2 -- src/Interpreters/ExternalLoaderXMLConfigRepository.cpp | 2 +- src/Interpreters/ExternalLoaderXMLConfigRepository.h | 2 +- src/Interpreters/IExternalLoaderConfigRepository.h | 2 +- src/Storages/StorageDictionary.cpp | 6 ------ src/Storages/StorageDictionary.h | 2 -- 10 files changed, 5 insertions(+), 30 deletions(-) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index ec429054475..3e162b1114f 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -169,7 +169,7 @@ public: private: struct FileInfo { - Poco::Timestamp last_update_time = 0; + std::optional last_update_time; bool in_use = true; // Whether the `FileInfo` should be destroyed because the correspondent file is deleted. Poco::AutoPtr file_contents; // Parsed contents of the file. std::unordered_map objects; @@ -268,7 +268,7 @@ private: // is updated, but in the same second). // The solution to this is probably switching to std::filesystem // -- the work is underway to do so. - if (update_time_from_repository == file_info.last_update_time) + if (update_time_from_repository && (update_time_from_repository == file_info.last_update_time)) { file_info.in_use = true; return false; diff --git a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp index 86f5a9ded0a..042992fcd2e 100644 --- a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp @@ -26,11 +26,6 @@ bool ExternalLoaderDictionaryStorageConfigRepository::exists(const std::string & return getName() == loadable_definition_name; } -Poco::Timestamp ExternalLoaderDictionaryStorageConfigRepository::getUpdateTime(const std::string &) -{ - return dictionary_storage.getUpdateTime(); -} - LoadablesConfigurationPtr ExternalLoaderDictionaryStorageConfigRepository::load(const std::string &) { return dictionary_storage.getConfiguration(); diff --git a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h index 06d2b0faf75..f357df990e6 100644 --- a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h +++ b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h @@ -19,8 +19,6 @@ public: bool exists(const std::string & loadable_definition_name) override; - Poco::Timestamp getUpdateTime(const std::string & loadable_definition_name) override; - LoadablesConfigurationPtr load(const std::string & loadable_definition_name) override; private: diff --git a/src/Interpreters/ExternalLoaderTempConfigRepository.cpp b/src/Interpreters/ExternalLoaderTempConfigRepository.cpp index 10fc61a2ed0..0a8fc170294 100644 --- a/src/Interpreters/ExternalLoaderTempConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderTempConfigRepository.cpp @@ -28,14 +28,6 @@ bool ExternalLoaderTempConfigRepository::exists(const String & path_) } -Poco::Timestamp ExternalLoaderTempConfigRepository::getUpdateTime(const String & path_) -{ - if (!exists(path_)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Loadable {} not found", path_); - return creation_time; -} - - LoadablesConfigurationPtr ExternalLoaderTempConfigRepository::load(const String & path_) { if (!exists(path_)) diff --git a/src/Interpreters/ExternalLoaderTempConfigRepository.h b/src/Interpreters/ExternalLoaderTempConfigRepository.h index 92816dcb7f2..a3c67b3eed6 100644 --- a/src/Interpreters/ExternalLoaderTempConfigRepository.h +++ b/src/Interpreters/ExternalLoaderTempConfigRepository.h @@ -18,14 +18,12 @@ public: std::set getAllLoadablesDefinitionNames() override; bool exists(const String & path) override; - Poco::Timestamp getUpdateTime(const String & path) override; LoadablesConfigurationPtr load(const String & path) override; private: String name; String path; LoadablesConfigurationPtr config; - Poco::Timestamp creation_time; }; } diff --git a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp index 8b9584f1242..a15f918f457 100644 --- a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp @@ -24,7 +24,7 @@ ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository( { } -Poco::Timestamp ExternalLoaderXMLConfigRepository::getUpdateTime(const std::string & definition_entity_name) +std::optional ExternalLoaderXMLConfigRepository::getUpdateTime(const std::string & definition_entity_name) { return FS::getModificationTimestamp(definition_entity_name); } diff --git a/src/Interpreters/ExternalLoaderXMLConfigRepository.h b/src/Interpreters/ExternalLoaderXMLConfigRepository.h index 25646d72db3..814817cad6f 100644 --- a/src/Interpreters/ExternalLoaderXMLConfigRepository.h +++ b/src/Interpreters/ExternalLoaderXMLConfigRepository.h @@ -26,7 +26,7 @@ public: bool exists(const std::string & definition_entity_name) override; /// Return xml-file modification time via stat call - Poco::Timestamp getUpdateTime(const std::string & definition_entity_name) override; + std::optional getUpdateTime(const std::string & definition_entity_name) override; /// May contain definition about several entities (several dictionaries in one .xml file) LoadablesConfigurationPtr load(const std::string & config_file) override; diff --git a/src/Interpreters/IExternalLoaderConfigRepository.h b/src/Interpreters/IExternalLoaderConfigRepository.h index 0d0c8acc01a..6ba4add86e2 100644 --- a/src/Interpreters/IExternalLoaderConfigRepository.h +++ b/src/Interpreters/IExternalLoaderConfigRepository.h @@ -37,7 +37,7 @@ public: virtual bool exists(const std::string & path) = 0; /// Returns entity last update time - virtual Poco::Timestamp getUpdateTime(const std::string & path) = 0; + virtual std::optional getUpdateTime(const std::string & /* path */) { return {}; } /// Load configuration from some concrete source to AbstractConfiguration virtual LoadablesConfigurationPtr load(const std::string & path) = 0; diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 0c6f6a25b9d..ee76522fe48 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -194,12 +194,6 @@ void StorageDictionary::removeDictionaryConfigurationFromRepository() remove_repository_callback.reset(); } -Poco::Timestamp StorageDictionary::getUpdateTime() const -{ - std::lock_guard lock(dictionary_config_mutex); - return update_time; -} - LoadablesConfigurationPtr StorageDictionary::getConfiguration() const { std::lock_guard lock(dictionary_config_mutex); diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index 995a0192269..5c6b2f6d8ad 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -92,7 +92,6 @@ public: void alter(const AlterCommands & params, ContextPtr alter_context, AlterLockHolder &) override; - Poco::Timestamp getUpdateTime() const; LoadablesConfigurationPtr getConfiguration() const; String getDictionaryName() const { return dictionary_name; } @@ -102,7 +101,6 @@ private: const Location location; mutable std::mutex dictionary_config_mutex; - Poco::Timestamp update_time; LoadablesConfigurationPtr configuration; scope_guard remove_repository_callback; From ea77c48c23da8472b4aa1d43ffb531a15fba2044 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 16 Mar 2024 19:49:26 +0100 Subject: [PATCH 0150/1165] Fix style check --- src/Interpreters/Cache/LRUFileCachePriority.cpp | 5 ++++- src/Interpreters/Cache/Metadata.h | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 65138fd70fc..2ba3de4667f 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -15,11 +15,11 @@ namespace CurrentMetrics namespace ProfileEvents { extern const Event FilesystemCacheEvictionSkippedFileSegments; - extern const Event FilesystemCacheEvictionSkippedEvictingFileSegments; extern const Event FilesystemCacheEvictionTries; extern const Event FilesystemCacheEvictMicroseconds; extern const Event FilesystemCacheEvictedBytes; extern const Event FilesystemCacheEvictedFileSegments; + extern const Event FilesystemCacheEvictionSkippedEvictingFileSegments; } namespace DB @@ -167,7 +167,10 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CachePriorityGuard } if (entry.isEvicting(lock)) + { + ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictionSkippedEvictingFileSegments); continue; + } auto locked_key = entry.key_metadata->tryLock(); if (!locked_key || entry.size == 0) diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 7c08bcbe80e..52be6368e49 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -19,6 +19,10 @@ using DownloadQueuePtr = std::shared_ptr; using FileSegmentsHolderPtr = std::unique_ptr; class CacheMetadata; +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} struct FileSegmentMetadata : private boost::noncopyable { From c35296b6ea5811c386a5d1f5a65abcc8b2b96091 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 17 Mar 2024 15:50:10 +0100 Subject: [PATCH 0151/1165] Fix --- src/Interpreters/Cache/LRUFileCachePriority.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 2ba3de4667f..6285dc684cf 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -168,6 +168,7 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CachePriorityGuard if (entry.isEvicting(lock)) { + ++it; ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictionSkippedEvictingFileSegments); continue; } From 7931c44ff2f788ca34f23e097ecb78077cadeddf Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 17 Mar 2024 17:40:43 +0100 Subject: [PATCH 0152/1165] Check static storage policy --- docker/test/stateful/run.sh | 12 ++++++------ src/Interpreters/Cache/EvictionCandidates.cpp | 5 ++++- src/Interpreters/Cache/EvictionCandidates.h | 2 +- src/Interpreters/Cache/FileCache.cpp | 8 ++++++-- src/Interpreters/Cache/SLRUFileCachePriority.cpp | 2 +- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 98da5988ad5..18ac909df54 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -33,12 +33,12 @@ fi echo "Using cache policy: $cache_policy" -if [ "$cache_policy" = "SLRU" ]; then - sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ - | sed "s|LRU|SLRU|" \ - > /etc/clickhouse-server/config.d/storage_conf.xml.tmp - mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml -fi +#if [ "$cache_policy" = "SLRU" ]; then +# sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ +# | sed "s|LRU|SLRU|" \ +# > /etc/clickhouse-server/config.d/storage_conf.xml.tmp +# mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml +#fi if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then # It is not needed, we will explicitly create tables on s3. diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index bfd986fd63f..7f77f14ee76 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -30,6 +30,8 @@ EvictionCandidates::~EvictionCandidates() void EvictionCandidates::add(const FileSegmentMetadataPtr & candidate, LockedKey & locked_key, const CachePriorityGuard::Lock & lock) { + chassert(lock.owns_lock()); + auto [it, inserted] = candidates.emplace(locked_key.getKey(), KeyCandidates{}); if (inserted) it->second.key_metadata = locked_key.getKeyMetadata(); @@ -39,7 +41,7 @@ void EvictionCandidates::add(const FileSegmentMetadataPtr & candidate, LockedKey ++candidates_size; } -void EvictionCandidates::evict(const CachePriorityGuard::Lock &) +void EvictionCandidates::evict() { if (candidates.empty()) return; @@ -86,6 +88,7 @@ void EvictionCandidates::evict(const CachePriorityGuard::Lock &) void EvictionCandidates::finalize(FileCacheQueryLimit::QueryContext * query_context, const CachePriorityGuard::Lock & lock) { + chassert(lock.owns_lock()); while (!queue_entries_to_invalidate.empty()) { auto iterator = queue_entries_to_invalidate.back(); diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index b1f4caedab6..19df0453971 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -11,7 +11,7 @@ public: void add(const FileSegmentMetadataPtr & candidate, LockedKey & locked_key, const CachePriorityGuard::Lock &); - void evict(const CachePriorityGuard::Lock &); + void evict(); void finalize(FileCacheQueryLimit::QueryContext * query_context, const CachePriorityGuard::Lock &); diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 480c1d00280..ffb56d6c937 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -783,6 +783,7 @@ bool FileCache::tryReserve( ProfileEventTimeIncrement watch(ProfileEvents::FilesystemCacheReserveMicroseconds); assertInitialized(); + auto cache_lock = tryLockCache(std::chrono::milliseconds(lock_wait_timeout_milliseconds)); if (!cache_lock) { @@ -841,7 +842,7 @@ bool FileCache::tryReserve( return false; /// Let's release cache lock if we are going to remove files from filesystem. - bool release_lock = eviction_candidates.size() > 0; + const bool release_lock = eviction_candidates.size() > 0; if (release_lock) cache_lock.unlock(); @@ -851,10 +852,13 @@ bool FileCache::tryReserve( try { /// Remove eviction candidates from filesystem. - eviction_candidates.evict(cache_lock); + eviction_candidates.evict(); } catch (...) { + if (release_lock) + cache_lock.lock(); + /// Invalidate queue entries if some succeeded to be removed. eviction_candidates.finalize(query_context.get(), cache_lock); throw; diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index aca59ede6c2..57c4a888025 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -217,7 +217,7 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach return; } /// Make space for "downgrade" candidates. - eviction_candidates.evict(lock); + eviction_candidates.evict(); eviction_candidates.finalize(nullptr, lock); } From 12289a9ebe07ce15937ceb4cffe8fe4cf033dcdd Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Sun, 17 Mar 2024 22:34:41 +0100 Subject: [PATCH 0153/1165] Add support of ParallelReadBuffer to ReadBufferFromAzureBlobStorage --- .../IO/ReadBufferFromAzureBlobStorage.cpp | 51 +++++++++++++++++++ src/Disks/IO/ReadBufferFromAzureBlobStorage.h | 4 ++ 2 files changed, 55 insertions(+) diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 8de977ef876..7cda8898a65 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -8,7 +8,9 @@ #include #include #include +#include +#include namespace ProfileEvents { @@ -262,6 +264,55 @@ size_t ReadBufferFromAzureBlobStorage::getFileSize() return *file_size; } +size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t range_begin, const std::function & progress_callback) const +{ + size_t initial_n = n; + + size_t sleep_time_with_backoff_milliseconds = 100; + auto handle_exception = [&, this](const auto & e, size_t i) + { + LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message); + if (i + 1 == max_single_download_retries) + throw; + + sleepForMilliseconds(sleep_time_with_backoff_milliseconds); + sleep_time_with_backoff_milliseconds *= 2; + }; + + Azure::Storage::Blobs::DownloadBlobOptions download_options; + download_options.Range = {static_cast(range_begin), range_begin+n}; + + for (size_t i = 0; i < max_single_download_retries; ++i) + { + size_t bytes_copied = 0; + try + { + auto download_response = blob_client->Download(download_options); + std::unique_ptr body_stream = std::move(download_response.Value.BodyStream); + auto length = body_stream->Length(); + char buffer[length]; + body_stream->Read(reinterpret_cast(buffer), length); + std::istringstream string_stream(String(static_cast(buffer),length)); + copyFromIStreamWithProgressCallback(string_stream, to, n, progress_callback, &bytes_copied); + + if (read_settings.remote_throttler) + read_settings.remote_throttler->add(bytes_copied, ProfileEvents::RemoteReadThrottlerBytes, ProfileEvents::RemoteReadThrottlerSleepMicroseconds); + + break; + } + catch (const Azure::Core::RequestFailedException & e) + { + handle_exception(e,i); + } + + range_begin += bytes_copied; + to += bytes_copied; + n -= bytes_copied; + } + + return initial_n; +} + } #endif diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.h b/src/Disks/IO/ReadBufferFromAzureBlobStorage.h index 68058b84a2f..d328195cc26 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.h @@ -44,6 +44,10 @@ public: size_t getFileSize() override; + size_t readBigAt(char * to, size_t n, size_t range_begin, const std::function & progress_callback) const override; + + bool supportsReadAt() override { return true; } + private: void initialize(); From fd9231a886ffb37db3657695b590b43d0fbd31ef Mon Sep 17 00:00:00 2001 From: Dan Wu Date: Mon, 18 Mar 2024 07:41:50 +0800 Subject: [PATCH 0154/1165] Add database = currentDatabase() condition to query test 03010_read_system_parts_table_test.sql The failed test suggest to add database = currentDatabase() condition to tests that queries on system.parts. --- .../0_stateless/03010_read_system_parts_table_test.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql index 6ec7e6212d5..8871822af4e 100644 --- a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql @@ -4,7 +4,7 @@ CREATE TABLE users (uid Int16, name String, age Int16) ENGINE=MergeTree ORDER BY INSERT INTO users VALUES (1231, 'John', 33); INSERT INTO users VALUES (6666, 'Ksenia', 48); -SELECT uuid, name from system.parts WHERE table = 'users'; +SELECT uuid, name from system.parts WHERE database = currentDatabase() AND table = 'users'; -SELECT table from system.parts WHERE uuid = '00000000-0000-0000-0000-000000000000'; +SELECT table from system.parts WHERE database = currentDatabase() AND uuid = '00000000-0000-0000-0000-000000000000'; DROP TABLE IF EXISTS users; From 1a61da1baefa5ae3f0f4d168a7357b4ce2221c82 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 02:16:52 +0100 Subject: [PATCH 0155/1165] Replace getFuzzerData with query text fuzzer in clickhouse-local --- CMakeLists.txt | 10 ++- programs/CMakeLists.txt | 4 - programs/local/CMakeLists.txt | 6 -- programs/local/LocalServer.cpp | 133 ++++++--------------------------- programs/main.cpp | 10 +-- src/Client/ClientBase.cpp | 25 +++++++ src/Client/ClientBase.h | 7 ++ src/Functions/CMakeLists.txt | 4 - 8 files changed, 66 insertions(+), 133 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c4e16eace2..da29a9c93e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,8 @@ if (ENABLE_FUZZING) # For codegen_select_fuzzer set (ENABLE_PROTOBUF 1) + + add_compile_definitions(FUZZING_MODE=1) endif() # Global libraries @@ -574,7 +576,7 @@ if (FUZZER) if (NOT(target_type STREQUAL "INTERFACE_LIBRARY" OR target_type STREQUAL "UTILITY")) target_compile_options(${target} PRIVATE "-fsanitize=fuzzer-no-link") endif() - if (target_type STREQUAL "EXECUTABLE" AND (target MATCHES ".+_fuzzer" OR target STREQUAL "clickhouse")) + if (target_type STREQUAL "EXECUTABLE" AND target MATCHES ".+_fuzzer") message(STATUS "${target} instrumented with fuzzer") target_link_libraries(${target} PUBLIC ch_contrib::fuzzer) # Add to fuzzers bundle @@ -583,6 +585,12 @@ if (FUZZER) get_target_property(target_bin_dir ${target} BINARY_DIR) add_custom_command(TARGET fuzzers POST_BUILD COMMAND mv "${target_bin_dir}/${target_bin_name}" "${CMAKE_CURRENT_BINARY_DIR}/programs/" VERBATIM) endif() + if (target STREQUAL "clickhouse") + message(STATUS "${target} instrumented with fuzzer") + target_link_libraries(${target} PUBLIC ch_contrib::fuzzer_no_main) + # Add to fuzzers bundle + add_dependencies(fuzzers ${target}) + endif() endif() endforeach() add_custom_command(TARGET fuzzers POST_BUILD COMMAND SRC=${CMAKE_SOURCE_DIR} BIN=${CMAKE_BINARY_DIR} OUT=${CMAKE_BINARY_DIR}/programs ${CMAKE_SOURCE_DIR}/tests/fuzz/build.sh VERBATIM) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index d945fdf4a6f..0d91de2dad8 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -268,10 +268,6 @@ if (ENABLE_TESTS) add_dependencies(clickhouse-bundle clickhouse-tests) endif() -if (ENABLE_FUZZING) - add_compile_definitions(FUZZING_MODE=1) -endif () - if (TARGET ch_contrib::protobuf) get_property(google_proto_files TARGET ch_contrib::protobuf PROPERTY google_proto_files) foreach (proto_file IN LISTS google_proto_files) diff --git a/programs/local/CMakeLists.txt b/programs/local/CMakeLists.txt index 565b67d0020..1aaa2859898 100644 --- a/programs/local/CMakeLists.txt +++ b/programs/local/CMakeLists.txt @@ -25,9 +25,3 @@ endif() # Always use internal readpassphrase target_link_libraries(clickhouse-local-lib PRIVATE readpassphrase) - -if (ENABLE_FUZZING) - add_compile_definitions(FUZZING_MODE=1) - set (WITH_COVERAGE ON) - target_link_libraries(clickhouse-local-lib PRIVATE ${LIB_FUZZING_ENGINE}) -endif () diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 167d361f5e3..4bb5d3a94b7 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -22,8 +21,6 @@ #include #include #include -#include -#include #include #include #include @@ -34,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -42,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -61,10 +56,6 @@ #include "config.h" -#if defined(FUZZING_MODE) - #include -#endif - #if USE_AZURE_BLOB_STORAGE # include #endif @@ -474,25 +465,10 @@ try } } -#if defined(FUZZING_MODE) - static bool first_time = true; - if (first_time) - { - - if (queries_files.empty() && queries.empty()) - { - std::cerr << "\033[31m" << "ClickHouse compiled in fuzzing mode." << "\033[0m" << std::endl; - std::cerr << "\033[31m" << "You have to provide a query with --query or --queries-file option." << "\033[0m" << std::endl; - std::cerr << "\033[31m" << "The query have to use function getFuzzerData() inside." << "\033[0m" << std::endl; - exit(1); - } - - is_interactive = false; -#else is_interactive = stdin_is_a_tty && (config().hasOption("interactive") || (queries.empty() && !config().has("table-structure") && queries_files.empty() && !config().has("table-file"))); -#endif + if (!is_interactive) { /// We will terminate process on error @@ -538,15 +514,13 @@ try connect(); -#ifdef FUZZING_MODE - first_time = false; - } -#endif - String initial_query = getInitialCreateTableQuery(); if (!initial_query.empty()) processQueryText(initial_query); +#if defined(FUZZING_MODE) + runLibFuzzer(); +#else if (is_interactive && !delayed_interactive) { runInteractive(); @@ -558,10 +532,8 @@ try if (delayed_interactive) runInteractive(); } - -#ifndef FUZZING_MODE - cleanup(); #endif + return Application::EXIT_OK; } catch (const DB::Exception & e) @@ -829,23 +801,11 @@ void LocalServer::processConfig() void LocalServer::printHelpMessage([[maybe_unused]] const OptionsDescription & options_description) { -#if defined(FUZZING_MODE) - std::cout << - "usage: clickhouse -- \n" - "Note: It is important not to use only one letter keys with single dash for \n" - "for clickhouse-local arguments. It may work incorrectly.\n" - - "ClickHouse is build with coverage guided fuzzer (libfuzzer) inside it.\n" - "You have to provide a query which contains getFuzzerData function.\n" - "This will take the data from fuzzing engine, pass it to getFuzzerData function and execute a query.\n" - "Each time the data will be different, and it will last until some segfault or sanitizer assertion is found. \n"; -#else std::cout << getHelpHeader() << "\n"; std::cout << options_description.main_description.value() << "\n"; std::cout << getHelpFooter() << "\n"; std::cout << "In addition, --param_name=value can be specified for substitution of parameters for parametrized queries.\n"; std::cout << "\nSee also: https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/\n"; -#endif } @@ -921,6 +881,7 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum for (int arg_num = 1; arg_num < argc; ++arg_num) { std::string_view arg = argv[arg_num]; + /// Parameter arg after underline. if (arg.starts_with("--param_")) { @@ -952,9 +913,27 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum arg = argv[arg_num]; addMultiquery(arg, common_arguments); } + else if (arg == "--") + { +#if defined(FUZZING_MODE) + fuzzer_argc = 1 + argc - arg_num; + fuzzer_argv = argv + arg_num; + break; +#endif + } else + { common_arguments.emplace_back(arg); + } } + +#if defined(FUZZING_MODE) + if (!fuzzer_argc) + { + fuzzer_argc = 1; + fuzzer_argv = argv; + } +#endif } } @@ -988,67 +967,3 @@ int mainEntryClickHouseLocal(int argc, char ** argv) return code ? code : 1; } } - -#if defined(FUZZING_MODE) - -// linked from programs/main.cpp -bool isClickhouseApp(const std::string & app_suffix, std::vector & argv); - -std::optional fuzz_app; - -extern "C" int LLVMFuzzerInitialize(int * pargc, char *** pargv) -{ - std::vector argv(*pargv, *pargv + (*pargc + 1)); - - /// As a user you can add flags to clickhouse binary in fuzzing mode as follows - /// clickhouse local -- - - char **p = &(*pargv)[1]; - - auto it = argv.begin() + 1; - for (; *it; ++it) - { - if (strcmp(*it, "--") == 0) - { - ++it; - break; - } - } - - while (*it) - { - if (strncmp(*it, "--", 2) != 0) - { - *(p++) = *it; - it = argv.erase(it); - } - else - ++it; - } - - *pargc = static_cast(p - &(*pargv)[0]); - *p = nullptr; - - /// Initialize clickhouse-local app - fuzz_app.emplace(); - fuzz_app->init(static_cast(argv.size() - 1), argv.data()); - - return 0; -} - - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size) -{ - try - { - auto input = String(reinterpret_cast(data), size); - DB::FunctionGetFuzzerData::update(input); - fuzz_app->run(); - } - catch (...) - { - } - - return 0; -} -#endif diff --git a/programs/main.cpp b/programs/main.cpp index 0a35594bd30..7162a18d764 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -67,7 +67,6 @@ namespace using MainFunc = int (*)(int, char**); -#if !defined(FUZZING_MODE) /// Add an item here to register new application std::pair clickhouse_applications[] = { @@ -111,7 +110,6 @@ int printHelp(int, char **) std::cerr << "clickhouse " << application.first << " [args] " << std::endl; return -1; } -#endif /// Add an item here to register a new short name std::pair clickhouse_short_names[] = @@ -284,7 +282,7 @@ struct Checker ; -#if !defined(FUZZING_MODE) && !defined(USE_MUSL) +#if !defined(USE_MUSL) /// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete. void checkHarmfulEnvironmentVariables(char ** argv) { @@ -446,13 +444,8 @@ extern "C" /// /// extern bool inside_main; /// class C { C() { assert(inside_main); } }; -#ifndef FUZZING_MODE bool inside_main = false; -#else -bool inside_main = true; -#endif -#if !defined(FUZZING_MODE) int main(int argc_, char ** argv_) { inside_main = true; @@ -510,4 +503,3 @@ int main(int argc_, char ** argv_) return exit_code; } -#endif diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index d561a64895b..da0807d0c26 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2631,6 +2631,31 @@ void ClientBase::runNonInteractive() } +#if defined(FUZZING_MODE) +extern "C" int LLVMFuzzerRunDriver(int * argc, char *** argv, int (*callback)(const uint8_t * data, size_t size)); +ClientBase * app; + +void ClientBase::runLibFuzzer() +{ + app = this; + LLVMFuzzerRunDriver(&fuzzer_argc, &fuzzer_argv, [](const uint8_t * data, size_t size) + { + try + { + String query(reinterpret_cast(data), size); + app->processQueryText(query); + } + catch (...) + { + return -1; + } + + return 0; + }); +} +#endif + + void ClientBase::clearTerminal() { /// Clear from cursor until end of screen. diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 7a9e9666e67..48a779d1757 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -78,6 +78,13 @@ protected: void runInteractive(); void runNonInteractive(); +#if defined(FUZZING_MODE) + int fuzzer_argc = 0; + char ** fuzzer_argv = nullptr; + + void runLibFuzzer(); +#endif + virtual bool processWithFuzzing(const String &) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Query processing with fuzzing is not implemented"); diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index ac3e3671ae0..bfb1fc92df8 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -142,10 +142,6 @@ list (APPEND OBJECT_LIBS $ Date: Mon, 18 Mar 2024 02:16:24 +0100 Subject: [PATCH 0156/1165] Fix error in clickhouse-client --- src/Parsers/parseQuery.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Parsers/parseQuery.cpp b/src/Parsers/parseQuery.cpp index 7aad0b010a5..51878efa706 100644 --- a/src/Parsers/parseQuery.cpp +++ b/src/Parsers/parseQuery.cpp @@ -92,7 +92,9 @@ void writeQueryWithHighlightedErrorPositions( } else { - size_t bytes_to_hilite = UTF8::seqLength(*current_position_to_hilite); + ssize_t bytes_to_hilite = UTF8::seqLength(*current_position_to_hilite); + if (bytes_to_hilite > end - current_position_to_hilite) + bytes_to_hilite = end - current_position_to_hilite; /// Bright on red background. out << "\033[41;1m"; From 2eb7de859d08a889db4d2e4f543a8c1699fcb5f4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 02:35:32 +0100 Subject: [PATCH 0157/1165] Correct arguments parsing --- programs/local/LocalServer.cpp | 16 ++++++---------- src/Client/ClientBase.cpp | 3 +++ src/Client/ClientBase.h | 4 +--- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 4bb5d3a94b7..f485e3880c1 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -878,6 +878,11 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector &, std::vector &) { +#if defined(FUZZING_MODE) + if (argc) + fuzzer_args.push_back(argv[0]); +#endif + for (int arg_num = 1; arg_num < argc; ++arg_num) { std::string_view arg = argv[arg_num]; @@ -916,8 +921,7 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum else if (arg == "--") { #if defined(FUZZING_MODE) - fuzzer_argc = 1 + argc - arg_num; - fuzzer_argv = argv + arg_num; + fuzzer_args.insert(fuzzer_args.end(), &argv[arg_num + 1], &argv[argc]); break; #endif } @@ -926,14 +930,6 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum common_arguments.emplace_back(arg); } } - -#if defined(FUZZING_MODE) - if (!fuzzer_argc) - { - fuzzer_argc = 1; - fuzzer_argv = argv; - } -#endif } } diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index da0807d0c26..c4cb6e1a5ad 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2638,6 +2638,9 @@ ClientBase * app; void ClientBase::runLibFuzzer() { app = this; + int fuzzer_argc = fuzzer_args.size(); + char ** fuzzer_argv = fuzzer_args.data(); + LLVMFuzzerRunDriver(&fuzzer_argc, &fuzzer_argv, [](const uint8_t * data, size_t size) { try diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 48a779d1757..c0052ade7aa 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -79,9 +79,7 @@ protected: void runNonInteractive(); #if defined(FUZZING_MODE) - int fuzzer_argc = 0; - char ** fuzzer_argv = nullptr; - + std::vector fuzzer_args; void runLibFuzzer(); #endif From bf85f7f368c09a11946ccba02d11b55ef71ffdf4 Mon Sep 17 00:00:00 2001 From: Zhuo Qiu Date: Mon, 18 Mar 2024 11:02:02 +0800 Subject: [PATCH 0158/1165] add docs & minor fix for lwd-aware merge settings --- .../settings/merge-tree-settings.md | 28 +++++++++++++++++++ src/Storages/MergeTree/IMergeTreeDataPart.cpp | 5 ++-- src/Storages/MergeTree/IMergeTreeDataPart.h | 3 +- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index c7e461d15ae..9ce705ff805 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -867,3 +867,31 @@ Default value: `Never` Persists virtual column `_block_number` on merges. Default value: false. + +## exclude_deleted_rows_for_part_size_in_merge {#exclude_deleted_rows_for_part_size_in_merge} + +If enabled, estimated actual size of data parts (i.e., excluding those rows that have been deleted through `DELETE FROM`) will be used when selecting parts to merge. Note that this behavior is only triggered for data parts affected by `DELETE FROM` executed after this setting is enabled. + +Possible values: + +- true, false + +Default value: false + +**See Also** + +- [load_existing_rows_count_for_old_parts](#load_existing_rows_count_for_old_parts) setting + +## load_existing_rows_count_for_old_parts {#load_existing_rows_count_for_old_parts} + +If enabled along with [exclude_deleted_rows_for_part_size_in_merge](#exclude_deleted_rows_for_part_size_in_merge), deleted rows count for existing data parts will be calculated during table starting up. + +Possible values: + +- true, false + +Default value: false + +**See Also** + +- [exclude_deleted_rows_for_part_size_in_merge](#exclude_deleted_rows_for_part_size_in_merge) setting diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 023202019e4..3ae3b678f37 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1349,8 +1349,9 @@ void IMergeTreeDataPart::loadExistingRowsCount() if (existing_rows_count.has_value()) return; - if (!rows_count || !storage.getSettings()->load_existing_rows_count_for_old_parts || !supportLightweightDeleteMutate() - || !hasLightweightDelete()) + if (!rows_count || !supportLightweightDeleteMutate() || !hasLightweightDelete() + || !storage.getSettings()->exclude_deleted_rows_for_part_size_in_merge + || !storage.getSettings()->load_existing_rows_count_for_old_parts) existing_rows_count = rows_count; else existing_rows_count = readExistingRowsCount(); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 8bd32e777bc..075a5c45710 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -674,7 +674,8 @@ private: /// For the older format version calculates rows count from the size of a column with a fixed size. void loadRowsCount(); - /// Load existing rows count from _row_exists column if load_existing_rows_count_for_old_parts is true. + /// Load existing rows count from _row_exists column + /// if load_existing_rows_count_for_old_parts and exclude_deleted_rows_for_part_size_in_merge are both enabled. void loadExistingRowsCount(); static void appendFilesOfRowsCount(Strings & files); From 248a8a60f1722414523a4ce19977cf7a68c636a6 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 18 Mar 2024 10:02:52 +0100 Subject: [PATCH 0159/1165] Fix style check --- src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 7cda8898a65..13f62c7ac25 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -292,7 +292,7 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran auto length = body_stream->Length(); char buffer[length]; body_stream->Read(reinterpret_cast(buffer), length); - std::istringstream string_stream(String(static_cast(buffer),length)); + std::istringstream string_stream(String(static_cast(buffer),length)); // STYLE_CHECK_ALLOW_STD_STRING_STREAM copyFromIStreamWithProgressCallback(string_stream, to, n, progress_callback, &bytes_copied); if (read_settings.remote_throttler) From 6b954fcfd42746e7c9f09329a89c0e31f329d41c Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 14 Mar 2024 11:31:53 +0000 Subject: [PATCH 0160/1165] Fix crash in ObjectJson parsing array with nulls --- src/DataTypes/ObjectUtils.cpp | 5 +++++ src/DataTypes/ObjectUtils.h | 2 +- .../queries/0_stateless/02287_type_object_convert.reference | 1 + tests/queries/0_stateless/02287_type_object_convert.sql | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index c565930f49a..1d697159b56 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -1001,6 +1001,11 @@ Field FieldVisitorFoldDimension::operator()(const Array & x) const return res; } +Field FieldVisitorFoldDimension::operator()(const Null &) const +{ + return Array(); +} + void setAllObjectsToDummyTupleType(NamesAndTypesList & columns) { for (auto & column : columns) diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 2bfcaae09ca..2b729dddc28 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -139,7 +139,7 @@ public: Field operator()(const Array & x) const; - Field operator()(const Null & x) const { return x; } + Field operator()(const Null & x) const; template Field operator()(const T & x) const diff --git a/tests/queries/0_stateless/02287_type_object_convert.reference b/tests/queries/0_stateless/02287_type_object_convert.reference index 501536f1f3e..af4407ee052 100644 --- a/tests/queries/0_stateless/02287_type_object_convert.reference +++ b/tests/queries/0_stateless/02287_type_object_convert.reference @@ -16,3 +16,4 @@ {"x":[[],[1,2]]} {"x":[[],[[1],[2]]]} {"x":[[],[[],[2]]]} +{"a.a":[[1],[]]} diff --git a/tests/queries/0_stateless/02287_type_object_convert.sql b/tests/queries/0_stateless/02287_type_object_convert.sql index 2bc08cf7fe1..21c1aba63d3 100644 --- a/tests/queries/0_stateless/02287_type_object_convert.sql +++ b/tests/queries/0_stateless/02287_type_object_convert.sql @@ -32,3 +32,4 @@ SELECT CAST('{"x" : [ 1 , [ 1 , 2] ]}', 'Object(\'json\')'); SELECT CAST('{"x" : [ {} , [ 1 , 2] ]}', 'Object(\'json\')'); SELECT CAST('{"x" : [ {} , [ 1 , [2]] ]}', 'Object(\'json\')'); SELECT CAST('{"x" : [ {} , [ {} , [2]] ]}', 'Object(\'json\')'); +SELECT CAST(' {"a": { "a": [ [1], null ] } }', 'Object(Nullable(\'json\'))'); From 9c9a4db2de3d04a1fce453c521a583264c4b4d06 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 15 Mar 2024 15:07:29 +0000 Subject: [PATCH 0161/1165] fix --- src/DataTypes/ObjectUtils.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 1d697159b56..1a64b29e82f 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -1001,8 +1001,11 @@ Field FieldVisitorFoldDimension::operator()(const Array & x) const return res; } -Field FieldVisitorFoldDimension::operator()(const Null &) const +Field FieldVisitorFoldDimension::operator()(const Null & x) const { + if (num_dimensions_to_fold == 0) + return x; + return Array(); } From 567a550149a37cfccc1db7b5b7a52f978a4c2e5b Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 15 Mar 2024 15:15:36 +0000 Subject: [PATCH 0162/1165] Safer Subcolumn::insert --- src/Columns/ColumnObject.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 25a62440629..0a4f90c2262 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -20,12 +20,12 @@ namespace DB namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; + extern const int ARGUMENT_OUT_OF_BOUND; extern const int DUPLICATE_COLUMN; + extern const int EXPERIMENTAL_FEATURE_ERROR; + extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_DIMENSIONS_MISMATCHED; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; - extern const int ARGUMENT_OUT_OF_BOUND; - extern const int EXPERIMENTAL_FEATURE_ERROR; } namespace @@ -334,7 +334,18 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) if (type_changed || info.need_convert) field = convertFieldToTypeOrThrow(field, *least_common_type.get()); - data.back()->insert(field); + if (!data.back()->tryInsert(field)) + { + /** Normalization of the field above is pretty complicated (it uses several FieldVisitors), + * so in the case of a bug, we may get mismatched types. + * The `IColumn::insert` method does not check the type of the inserted field, and it can lead to a segmentation fault. + * Therefore, we use the safer `tryInsert` method to get an exception instead of a segmentation fault. + */ + throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, + "Cannot insert field {} to column {}", + field.dump(), data.back()->dumpStructure()); + } + ++num_rows; } From 9065f34534a78ebc09cd0091ad18e8e5cac7ba46 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 12:26:05 +0100 Subject: [PATCH 0163/1165] Fix infinite loop in function `hop` --- src/Functions/FunctionsTimeWindow.cpp | 205 +++++++++++------- src/Functions/FunctionsTimeWindow.h | 35 --- .../03013_hop_infinite_loop.reference | 0 .../0_stateless/03013_hop_infinite_loop.sql | 2 + 4 files changed, 126 insertions(+), 116 deletions(-) create mode 100644 tests/queries/0_stateless/03013_hop_infinite_loop.reference create mode 100644 tests/queries/0_stateless/03013_hop_infinite_loop.sql diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index 017661231f5..91eb70fdd66 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -27,87 +27,124 @@ namespace ErrorCodes namespace { - std::tuple - dispatchForIntervalColumns(const ColumnWithTypeAndName & interval_column, const String & function_name) - { - const auto * interval_type = checkAndGetDataType(interval_column.type.get()); - if (!interval_type) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", - interval_column.name, function_name); - const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); - if (!interval_column_const_int64) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", - interval_column.name, function_name); - Int64 num_units = interval_column_const_int64->getValue(); - if (num_units <= 0) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Value for column {} of function {} must be positive", - interval_column.name, function_name); - return {interval_type->getKind(), num_units}; - } +std::tuple +dispatchForIntervalColumns(const ColumnWithTypeAndName & interval_column, const String & function_name) +{ + const auto * interval_type = checkAndGetDataType(interval_column.type.get()); + if (!interval_type) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", + interval_column.name, function_name); + const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); + if (!interval_column_const_int64) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", + interval_column.name, function_name); + Int64 num_units = interval_column_const_int64->getValue(); + if (num_units <= 0) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Value for column {} of function {} must be positive", + interval_column.name, function_name); - ColumnPtr executeWindowBound(const ColumnPtr & column, int index, const String & function_name) + return {interval_type->getKind(), num_units}; +} + +ColumnPtr executeWindowBound(const ColumnPtr & column, int index, const String & function_name) +{ + if (const ColumnTuple * col_tuple = checkAndGetColumn(column.get()); col_tuple) { - if (const ColumnTuple * col_tuple = checkAndGetColumn(column.get()); col_tuple) - { - if (!checkColumn>(*col_tuple->getColumnPtr(index))) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " - "Must be a Tuple(DataTime, DataTime)", function_name); - return col_tuple->getColumnPtr(index); - } - else - { + if (!checkColumn>(*col_tuple->getColumnPtr(index))) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " - "Must be Tuple", function_name); - } + "Must be a Tuple(DataTime, DataTime)", function_name); + return col_tuple->getColumnPtr(index); } - - void checkFirstArgument(const ColumnWithTypeAndName & argument, const String & function_name) + else { - if (!isDateTime(argument.type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "Should be a date with time", argument.type->getName(), function_name); - } - - void checkIntervalArgument(const ColumnWithTypeAndName & argument, const String & function_name, IntervalKind & interval_kind, bool & result_type_is_date) - { - const auto * interval_type = checkAndGetDataType(argument.type.get()); - if (!interval_type) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "Should be an interval of time", argument.type->getName(), function_name); - interval_kind = interval_type->getKind(); - result_type_is_date = (interval_type->getKind() == IntervalKind::Kind::Year) || (interval_type->getKind() == IntervalKind::Kind::Quarter) - || (interval_type->getKind() == IntervalKind::Kind::Month) || (interval_type->getKind() == IntervalKind::Kind::Week); - } - - void checkIntervalArgument(const ColumnWithTypeAndName & argument, const String & function_name, bool & result_type_is_date) - { - IntervalKind interval_kind; - checkIntervalArgument(argument, function_name, interval_kind, result_type_is_date); - } - - void checkTimeZoneArgument( - const ColumnWithTypeAndName & argument, - const String & function_name) - { - if (!WhichDataType(argument.type).isString()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "This argument is optional and must be a constant string with timezone name", - argument.type->getName(), function_name); - } - - bool checkIntervalOrTimeZoneArgument(const ColumnWithTypeAndName & argument, const String & function_name, IntervalKind & interval_kind, bool & result_type_is_date) - { - if (WhichDataType(argument.type).isString()) - { - checkTimeZoneArgument(argument, function_name); - return false; - } - checkIntervalArgument(argument, function_name, interval_kind, result_type_is_date); - return true; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " + "Must be Tuple", function_name); } } +void checkFirstArgument(const ColumnWithTypeAndName & argument, const String & function_name) +{ + if (!isDateTime(argument.type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " + "Should be a date with time", argument.type->getName(), function_name); +} + +void checkIntervalArgument(const ColumnWithTypeAndName & argument, const String & function_name, IntervalKind & interval_kind, bool & result_type_is_date) +{ + const auto * interval_type = checkAndGetDataType(argument.type.get()); + if (!interval_type) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " + "Should be an interval of time", argument.type->getName(), function_name); + interval_kind = interval_type->getKind(); + result_type_is_date = (interval_type->getKind() == IntervalKind::Kind::Year) || (interval_type->getKind() == IntervalKind::Kind::Quarter) + || (interval_type->getKind() == IntervalKind::Kind::Month) || (interval_type->getKind() == IntervalKind::Kind::Week); +} + +void checkIntervalArgument(const ColumnWithTypeAndName & argument, const String & function_name, bool & result_type_is_date) +{ + IntervalKind interval_kind; + checkIntervalArgument(argument, function_name, interval_kind, result_type_is_date); +} + +void checkTimeZoneArgument( + const ColumnWithTypeAndName & argument, + const String & function_name) +{ + if (!WhichDataType(argument.type).isString()) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " + "This argument is optional and must be a constant string with timezone name", + argument.type->getName(), function_name); +} + +bool checkIntervalOrTimeZoneArgument(const ColumnWithTypeAndName & argument, const String & function_name, IntervalKind & interval_kind, bool & result_type_is_date) +{ + if (WhichDataType(argument.type).isString()) + { + checkTimeZoneArgument(argument, function_name); + return false; + } + checkIntervalArgument(argument, function_name, interval_kind, result_type_is_date); + return true; +} + +template +struct TimeWindowImpl +{ + static constexpr auto name = "UNKNOWN"; + + static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name); + + static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name); +}; + +template +class FunctionTimeWindow : public IFunction +{ +public: + static constexpr auto name = TimeWindowImpl::name; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + String getName() const override { return name; } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override; +}; + +using FunctionTumble = FunctionTimeWindow; +using FunctionTumbleStart = FunctionTimeWindow; +using FunctionTumbleEnd = FunctionTimeWindow; +using FunctionHop = FunctionTimeWindow; +using FunctionWindowId = FunctionTimeWindow; +using FunctionHopStart = FunctionTimeWindow; +using FunctionHopEnd = FunctionTimeWindow; + + template <> struct TimeWindowImpl { @@ -393,6 +430,9 @@ struct TimeWindowImpl wstart = AddTime::execute(wend, -window_num_units, time_zone); ToType wend_latest; + if (wstart > wend) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time overflow in function {}", name); + do { wend_latest = wend; @@ -414,7 +454,7 @@ struct TimeWindowImpl { static constexpr auto name = "windowID"; - [[maybe_unused]] static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name) + static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name) { bool result_type_is_date; IntervalKind interval_kind_1; @@ -456,8 +496,7 @@ struct TimeWindowImpl return std::make_shared(); } - [[maybe_unused]] static ColumnPtr - dispatchForHopColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForHopColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { const auto & time_column = arguments[0]; const auto & hop_interval_column = arguments[1]; @@ -527,6 +566,9 @@ struct TimeWindowImpl ToType wend = AddTime::execute(wstart, hop_num_units, time_zone); ToType wend_latest; + if (wstart > wend) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time overflow in function {}", name); + do { wend_latest = wend; @@ -538,14 +580,13 @@ struct TimeWindowImpl return end; } - [[maybe_unused]] static ColumnPtr - dispatchForTumbleColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForTumbleColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { ColumnPtr column = TimeWindowImpl::dispatchForColumns(arguments, function_name); return executeWindowBound(column, 1, function_name); } - [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { if (arguments.size() == 2) return dispatchForTumbleColumns(arguments, function_name); @@ -585,7 +626,7 @@ struct TimeWindowImpl } } - [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { const auto & time_column = arguments[0]; const auto which_type = WhichDataType(time_column.type); @@ -608,12 +649,12 @@ struct TimeWindowImpl { static constexpr auto name = "hopEnd"; - [[maybe_unused]] static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name) + static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name) { return TimeWindowImpl::getReturnType(arguments, function_name); } - [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { const auto & time_column = arguments[0]; const auto which_type = WhichDataType(time_column.type); @@ -644,6 +685,8 @@ ColumnPtr FunctionTimeWindow::executeImpl(const ColumnsWithTypeAndName & a return TimeWindowImpl::dispatchForColumns(arguments, name); } +} + REGISTER_FUNCTION(TimeWindow) { factory.registerFunction(); diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index 658963cb69c..ab8deef08c0 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -156,39 +156,4 @@ template <> \ ADD_SUBSECONDS(Nanosecond, 9) #undef ADD_SUBSECONDS -template -struct TimeWindowImpl -{ - static constexpr auto name = "UNKNOWN"; - - static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name); - - static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name); -}; - -template -class FunctionTimeWindow : public IFunction -{ -public: - static constexpr auto name = TimeWindowImpl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override { return name; } - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } - - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override; -}; - -using FunctionTumble = FunctionTimeWindow; -using FunctionTumbleStart = FunctionTimeWindow; -using FunctionTumbleEnd = FunctionTimeWindow; -using FunctionHop = FunctionTimeWindow; -using FunctionWindowId = FunctionTimeWindow; -using FunctionHopStart = FunctionTimeWindow; -using FunctionHopEnd = FunctionTimeWindow; } diff --git a/tests/queries/0_stateless/03013_hop_infinite_loop.reference b/tests/queries/0_stateless/03013_hop_infinite_loop.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03013_hop_infinite_loop.sql b/tests/queries/0_stateless/03013_hop_infinite_loop.sql new file mode 100644 index 00000000000..bdac87f5af1 --- /dev/null +++ b/tests/queries/0_stateless/03013_hop_infinite_loop.sql @@ -0,0 +1,2 @@ +SET allow_experimental_window_view = 1; +SELECT hop(toDateTime(0), INTERVAL 1 DAY, INTERVAL 3 DAY); -- { serverError BAD_ARGUMENTS } From 8434f049755ff8ea438d6d8c3cef1ad8f70cb2d4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 12:40:49 +0100 Subject: [PATCH 0164/1165] Fix style --- src/Functions/FunctionsTimeWindow.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index 91eb70fdd66..97608db51e8 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -23,6 +23,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; extern const int SYNTAX_ERROR; + extern const int BAD_ARGUMENTS; } namespace From 97aca12830f98890a6e2d71b2c7e9ce149d72cdb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 12:58:03 +0100 Subject: [PATCH 0165/1165] Fix crash in window view --- src/Functions/FunctionsTimeWindow.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index 97608db51e8..be51978499b 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -48,11 +48,12 @@ dispatchForIntervalColumns(const ColumnWithTypeAndName & interval_column, const return {interval_type->getKind(), num_units}; } -ColumnPtr executeWindowBound(const ColumnPtr & column, int index, const String & function_name) +ColumnPtr executeWindowBound(const ColumnPtr & column, size_t index, const String & function_name) { + chassert(index == 0 || index == 1); if (const ColumnTuple * col_tuple = checkAndGetColumn(column.get()); col_tuple) { - if (!checkColumn>(*col_tuple->getColumnPtr(index))) + if (index >= col_tuple->tupleSize() || !checkColumn>(*col_tuple->getColumnPtr(index))) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " "Must be a Tuple(DataTime, DataTime)", function_name); return col_tuple->getColumnPtr(index); From 9fa449fc75d1a9b69aa59737b5dd4fe0bc25dace Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 12:59:09 +0100 Subject: [PATCH 0166/1165] Add a test --- tests/queries/0_stateless/03014_window_view_crash.reference | 0 tests/queries/0_stateless/03014_window_view_crash.sql | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/03014_window_view_crash.reference create mode 100644 tests/queries/0_stateless/03014_window_view_crash.sql diff --git a/tests/queries/0_stateless/03014_window_view_crash.reference b/tests/queries/0_stateless/03014_window_view_crash.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03014_window_view_crash.sql b/tests/queries/0_stateless/03014_window_view_crash.sql new file mode 100644 index 00000000000..ff9835521f5 --- /dev/null +++ b/tests/queries/0_stateless/03014_window_view_crash.sql @@ -0,0 +1,2 @@ +SET allow_experimental_window_view = 1; +SELECT hopEnd(tuple([[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]], [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]], [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]])::Tuple(MultiPolygon)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } From 9850522be3541f08b8e8fc9bb85a2271483d66c9 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 18 Mar 2024 13:25:33 +0100 Subject: [PATCH 0167/1165] Add debug logging --- src/Interpreters/Cache/FileCache.cpp | 5 +++-- tests/config/install.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index ffb56d6c937..bc8236f37fb 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -792,9 +792,10 @@ bool FileCache::tryReserve( } LOG_TEST( - log, "Trying to reserve space ({} bytes) for {}:{}, current usage {}/{}", + log, "Trying to reserve space ({} bytes) for {}:{}, current usage {}/{} in size, {}/{} in elements", size, file_segment.key(), file_segment.offset(), - main_priority->getSize(cache_lock), main_priority->getSizeLimit(cache_lock)); + main_priority->getSize(cache_lock), main_priority->getSizeLimit(cache_lock), + main_priority->getElementsCount(cache_lock), main_priority->getElementsLimit(cache_lock)); /// In case of per query cache limit (by default disabled), we add/remove entries from both /// (main_priority and query_priority) priority queues, but iterate entries in order of query_priority, diff --git a/tests/config/install.sh b/tests/config/install.sh index 467636cfa40..06986871bd2 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -45,7 +45,7 @@ ln -sf $SRC_PATH/config.d/transactions.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/encryption.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/CORS.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/zookeeper_log.xml $DEST_SERVER_PATH/config.d/ -ln -sf $SRC_PATH/config.d/logger_trace.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/logger_test.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/named_collection.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/ssl_certs.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/filesystem_cache_log.xml $DEST_SERVER_PATH/config.d/ From eada7e8d29aa3d0a595125d70ceb5a7c20c1272f Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 18 Mar 2024 13:18:58 +0000 Subject: [PATCH 0168/1165] Fix server version --- tests/ci/stress.py | 2 +- tests/clickhouse-test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/stress.py b/tests/ci/stress.py index b0076449a53..e04f06c951b 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -66,7 +66,7 @@ def get_options(i: int, upgrade_check: bool) -> str: if random.random() < 0.3: client_options.append(f"http_make_head_request={random.randint(0, 1)}") - # TODO: After release 23.3 use ignore-drop-queries-probability for both + # TODO: After release 24.3 use ignore-drop-queries-probability for both # stress test and upgrade check if not upgrade_check: client_options.append("ignore-drop-queries-probability=0.5") diff --git a/tests/clickhouse-test b/tests/clickhouse-test index c457ae9a303..83bf2f59131 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -3170,7 +3170,7 @@ def parse_args(): help="Do not run shard related tests", ) - # TODO: Remove upgrade-check option after release 23.3 and use + # TODO: Remove upgrade-check option after release 24.3 and use # ignore-drop-queries-probability option in stress.py as in stress tests group.add_argument( "--upgrade-check", From 64b5acf0e47b65f616dce2427a57014e1e80d534 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 14 Mar 2024 18:38:27 +0100 Subject: [PATCH 0169/1165] Code cleanup --- .../ClusterProxy/SelectStreamFactory.h | 3 -- .../OptimizeShardingKeyRewriteInVisitor.cpp | 28 +++++++++---------- .../OptimizeShardingKeyRewriteInVisitor.h | 2 +- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 760281284fd..2e7b2445c6b 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -44,8 +43,6 @@ ASTPtr rewriteSelectQuery( using ColumnsDescriptionByShardNum = std::unordered_map; using AdditionalShardFilterGenerator = std::function; -using MissingObjectList = std::vector>; - class SelectStreamFactory { public: diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 0df6fcee229..84279282d92 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -1,19 +1,17 @@ -#include -#include -#include -#include -#include -#include -#include #include -#include "Analyzer/ColumnNode.h" -#include "Analyzer/ConstantNode.h" -#include "Analyzer/FunctionNode.h" -#include "Analyzer/IQueryTreeNode.h" -#include "Analyzer/InDepthQueryTreeVisitor.h" -#include "Analyzer/Utils.h" -#include "DataTypes/IDataType.h" -#include "Interpreters/Context_fwd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace { diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h index d202609160b..1b2d5a8fc15 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include "Analyzer/IQueryTreeNode.h" namespace DB { From 08d64f9e118329232a2149ff21e6ad47ffd37e1c Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 15 Mar 2024 00:42:12 +0100 Subject: [PATCH 0170/1165] Get rid of modifying a shared AbstractConfiguration (between StorageDictionary and ExternalDictionariesLoader). Always create a new configuration when we need to modify it. --- src/Common/Config/ConfigHelper.cpp | 29 +++++++++++++++++++++++++++++ src/Common/Config/ConfigHelper.h | 13 ++++++++----- src/Storages/StorageDictionary.cpp | 13 +++++++++---- src/Storages/StorageDictionary.h | 2 +- 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/Common/Config/ConfigHelper.cpp b/src/Common/Config/ConfigHelper.cpp index 69fe42de86c..6de63fe78d7 100644 --- a/src/Common/Config/ConfigHelper.cpp +++ b/src/Common/Config/ConfigHelper.cpp @@ -1,5 +1,7 @@ #include #include +#include + namespace DB { @@ -7,6 +9,33 @@ namespace DB namespace ConfigHelper { +namespace +{ + void cloneImpl(Poco::Util::AbstractConfiguration & dest, const Poco::Util::AbstractConfiguration & src, const std::string & prefix = "") + { + std::vector keys; + src.keys(prefix, keys); + if (!keys.empty()) + { + std::string prefix_with_dot = prefix + "."; + for (const auto & key : keys) + cloneImpl(dest, src, prefix_with_dot + key); + } + else if (!prefix.empty()) + { + dest.setString(prefix, src.getRawString(prefix)); + } + } +} + + +Poco::AutoPtr clone(const Poco::Util::AbstractConfiguration & src) +{ + Poco::AutoPtr res(new Poco::Util::XMLConfiguration()); + cloneImpl(*res, src); + return res; +} + bool getBool(const Poco::Util::AbstractConfiguration & config, const std::string & key, bool default_, bool empty_as) { if (!config.has(key)) diff --git a/src/Common/Config/ConfigHelper.h b/src/Common/Config/ConfigHelper.h index a1b630fa726..4f13bc4ad78 100644 --- a/src/Common/Config/ConfigHelper.h +++ b/src/Common/Config/ConfigHelper.h @@ -1,18 +1,21 @@ #pragma once +#include #include -namespace Poco + +namespace Poco::Util { - namespace Util - { - class AbstractConfiguration; - } + class AbstractConfiguration; } + namespace DB::ConfigHelper { +/// Clones a configuration. +Poco::AutoPtr clone(const Poco::Util::AbstractConfiguration & src); + /// The behavior is like `config.getBool(key, default_)`, /// except when the tag is empty (aka. self-closing), `empty_as` will be used instead of throwing Poco::Exception. bool getBool(const Poco::Util::AbstractConfiguration & config, const std::string & key, bool default_ = false, bool empty_as = true); diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index ee76522fe48..414c66e9522 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -221,13 +222,17 @@ void StorageDictionary::renameInMemory(const StorageID & new_table_id) { std::lock_guard lock(dictionary_config_mutex); + auto new_configuration = ConfigHelper::clone(*configuration); + + new_configuration->setString("dictionary.database", new_table_id.database_name); + new_configuration->setString("dictionary.name", new_table_id.table_name); - configuration->setString("dictionary.database", new_table_id.database_name); - configuration->setString("dictionary.name", new_table_id.table_name); if (move_to_atomic) - configuration->setString("dictionary.uuid", toString(new_table_id.uuid)); + new_configuration->setString("dictionary.uuid", toString(new_table_id.uuid)); else if (move_to_ordinary) - configuration->remove("dictionary.uuid"); + new_configuration->remove("dictionary.uuid"); + + configuration = new_configuration; } const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index 5c6b2f6d8ad..a414e4586d9 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -101,7 +101,7 @@ private: const Location location; mutable std::mutex dictionary_config_mutex; - LoadablesConfigurationPtr configuration; + LoadablesConfigurationPtr configuration TSA_GUARDED_BY(dictionary_config_mutex); scope_guard remove_repository_callback; From 5c60b1fab17823bd865394a54003336e5e0965a6 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 15 Mar 2024 02:52:30 +0100 Subject: [PATCH 0171/1165] Remove unnecessary reloading while renaming a dictionary in Atomic database. --- .../AbstractConfigurationComparison.cpp | 91 ++++++++++++------- .../Config/AbstractConfigurationComparison.h | 6 ++ .../ExternalDictionariesLoader.cpp | 14 +++ src/Interpreters/ExternalDictionariesLoader.h | 3 + src/Interpreters/ExternalLoader.cpp | 18 +++- src/Interpreters/ExternalLoader.h | 4 + 6 files changed, 98 insertions(+), 38 deletions(-) diff --git a/src/Common/Config/AbstractConfigurationComparison.cpp b/src/Common/Config/AbstractConfigurationComparison.cpp index c00a1efeebe..e983bd32f51 100644 --- a/src/Common/Config/AbstractConfigurationComparison.cpp +++ b/src/Common/Config/AbstractConfigurationComparison.cpp @@ -19,6 +19,56 @@ namespace result += subkey; return result; } + + bool isSameConfigurationImpl(const Poco::Util::AbstractConfiguration & left, const String & left_key, + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set * ignore_keys) + { + if (&left == &right && left_key == right_key) + return true; + + /// Get the subkeys of the left and right configurations. + Poco::Util::AbstractConfiguration::Keys left_subkeys; + Poco::Util::AbstractConfiguration::Keys right_subkeys; + left.keys(left_key, left_subkeys); + right.keys(right_key, right_subkeys); + + if (ignore_keys) + { + std::erase_if(left_subkeys, [&](const String & key) { return ignore_keys->contains(key); }); + std::erase_if(right_subkeys, [&](const String & key) { return ignore_keys->contains(key); }); + } + + /// Check that the right configuration has the same set of subkeys as the left configuration. + if (left_subkeys.size() != right_subkeys.size()) + return false; + + if (left_subkeys.empty()) + { + if (left.hasProperty(left_key)) + { + return right.hasProperty(right_key) && (left.getRawString(left_key) == right.getRawString(right_key)); + } + else + { + return !right.hasProperty(right_key); + } + } + else + { + /// Go through all the subkeys and compare corresponding parts of the configurations. + std::unordered_set left_subkeys_set{left_subkeys.begin(), left_subkeys.end()}; + for (const auto & subkey : right_subkeys) + { + if (!left_subkeys_set.contains(subkey)) + return false; + + if (!isSameConfiguration(left, concatKeyAndSubKey(left_key, subkey), right, concatKeyAndSubKey(right_key, subkey))) + return false; + } + return true; + } + } } @@ -52,41 +102,14 @@ bool isSameConfigurationWithMultipleKeys(const Poco::Util::AbstractConfiguration bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, const Poco::Util::AbstractConfiguration & right, const String & right_key) { - if (&left == &right && left_key == right_key) - return true; + return isSameConfigurationImpl(left, left_key, right, right_key, /* ignore_keys= */ nullptr); +} - bool has_property = left.hasProperty(left_key); - if (has_property != right.hasProperty(right_key)) - return false; - if (has_property) - { - /// The left and right configurations contains values so we can compare them. - if (left.getRawString(left_key) != right.getRawString(right_key)) - return false; - } - - /// Get the subkeys of the left and right configurations. - Poco::Util::AbstractConfiguration::Keys subkeys; - left.keys(left_key, subkeys); - - { - /// Check that the right configuration has the same set of subkeys as the left configuration. - Poco::Util::AbstractConfiguration::Keys right_subkeys; - right.keys(right_key, right_subkeys); - std::unordered_set left_subkeys{subkeys.begin(), subkeys.end()}; - if ((left_subkeys.size() != right_subkeys.size()) || (left_subkeys.size() != subkeys.size())) - return false; - for (const auto & right_subkey : right_subkeys) - if (!left_subkeys.contains(right_subkey)) - return false; - } - - /// Go through all the subkeys and compare corresponding parts of the configurations. - for (const auto & subkey : subkeys) - if (!isSameConfiguration(left, concatKeyAndSubKey(left_key, subkey), right, concatKeyAndSubKey(right_key, subkey))) - return false; - - return true; +bool isSameConfigurationIgnoringKeys(const Poco::Util::AbstractConfiguration & left, const String & left_key, + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set & ignore_keys) +{ + return isSameConfigurationImpl(left, left_key, right, right_key, !ignore_keys.empty() ? &ignore_keys : nullptr); } } diff --git a/src/Common/Config/AbstractConfigurationComparison.h b/src/Common/Config/AbstractConfigurationComparison.h index 75e0b77b1ca..edaff8d5363 100644 --- a/src/Common/Config/AbstractConfigurationComparison.h +++ b/src/Common/Config/AbstractConfigurationComparison.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace Poco::Util { @@ -33,6 +34,11 @@ namespace DB bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, const Poco::Util::AbstractConfiguration & right, const String & right_key); + /// Returns true if specified subviews of the two configurations contains the same keys and values, but without checking specified keys. + bool isSameConfigurationIgnoringKeys(const Poco::Util::AbstractConfiguration & left, const String & left_key, + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set & ignore_keys); + inline bool operator==(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right) { return isSameConfiguration(left, right); diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 760f94d18c2..0591a6232d0 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "config.h" @@ -41,6 +42,19 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( return DictionaryFactory::instance().create(name, config, key_in_config, getContext(), created_from_ddl); } +bool ExternalDictionariesLoader::doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & old_config, const String & old_key_in_config, + const Poco::Util::AbstractConfiguration & new_config, const String & new_key_in_config) const +{ + /// If the database is atomic then a dictionary can be renamed without reloading. + if (!old_config.getString(old_key_in_config + ".uuid", "").empty() && !new_config.getString(new_key_in_config + ".uuid", "").empty()) + { + static const std::unordered_set ignore_keys{"name", "database"}; + bool only_name_or_database_may_differ = isSameConfigurationIgnoringKeys(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); + return !only_name_or_database_may_differ; + } + return true; +} + void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const { IDictionary & dict = static_cast(object); diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h index 3c226f998be..f3aa8fa5f8e 100644 --- a/src/Interpreters/ExternalDictionariesLoader.h +++ b/src/Interpreters/ExternalDictionariesLoader.h @@ -43,6 +43,9 @@ protected: LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string & repository_name) const override; + bool doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & old_config, const String & old_key_in_config, + const Poco::Util::AbstractConfiguration & new_config, const String & new_key_in_config) const override; + void updateObjectFromConfigWithoutReloading( IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const override; diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 3e162b1114f..6ab083f9fb7 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -88,6 +88,7 @@ namespace } }; + using DoesConfigChangeRequiresReloadingObjectFunction = std::function; using UpdateObjectFromConfigWithoutReloadingFunction = std::function; } @@ -402,10 +403,12 @@ public: LoadingDispatcher( const CreateObjectFunction & create_object_function_, + const DoesConfigChangeRequiresReloadingObjectFunction & does_config_change_requires_reloading_object_, const UpdateObjectFromConfigWithoutReloadingFunction & update_object_from_config_without_reloading_, const String & type_name_, LoggerPtr log_) : create_object(create_object_function_) + , does_config_change_requires_reloading_object(does_config_change_requires_reloading_object_) , update_object_from_config_without_reloading(update_object_from_config_without_reloading_) , type_name(type_name_) , log(log_) @@ -472,10 +475,14 @@ public: if (info.triedToLoad()) { - /// The object has been tried to load before, so it is currently in use or was in use - /// and we should try to reload it with the new config. - LOG_TRACE(log, "Will reload '{}' because its configuration has been changed and there were attempts to load it before", name); - startLoading(info, true); + bool config_change_requires_reloading = does_config_change_requires_reloading_object(*previous_config->config, previous_config->key_in_config, *new_config->config, new_config->key_in_config); + if (config_change_requires_reloading) + { + /// The object has been tried to load before, so it is currently in use or was in use + /// and we should try to reload it with the new config. + LOG_TRACE(log, "Will reload '{}' because its configuration has been changed and there were attempts to load it before", name); + startLoading(info, true); + } } } } @@ -1204,6 +1211,7 @@ private: } const CreateObjectFunction create_object; + const DoesConfigChangeRequiresReloadingObjectFunction does_config_change_requires_reloading_object; const UpdateObjectFromConfigWithoutReloadingFunction update_object_from_config_without_reloading; const String type_name; LoggerPtr log; @@ -1290,6 +1298,8 @@ ExternalLoader::ExternalLoader(const String & type_name_, LoggerPtr log_) : config_files_reader(std::make_unique(type_name_, log_)) , loading_dispatcher(std::make_unique( [this](auto && a, auto && b, auto && c) { return createObject(a, b, c); }, + [this](const Poco::Util::AbstractConfiguration & config_1, const String & key_in_config_1, const Poco::Util::AbstractConfiguration & config_2, const String & key_in_config_2) + { return doesConfigChangeRequiresReloadingObject(config_1, key_in_config_1, config_2, key_in_config_2); }, [this](const IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) { return updateObjectFromConfigWithoutReloading(const_cast(object), config, key_in_config); }, type_name_, diff --git a/src/Interpreters/ExternalLoader.h b/src/Interpreters/ExternalLoader.h index 279157b6d51..781d8ca9e7b 100644 --- a/src/Interpreters/ExternalLoader.h +++ b/src/Interpreters/ExternalLoader.h @@ -213,6 +213,10 @@ public: protected: virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0; + /// Returns whether the object must be reloaded after a specified change in its configuration. + virtual bool doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & /* old_config */, const String & /* old_key_in_config */, + const Poco::Util::AbstractConfiguration & /* new_config */, const String & /* new_key_in_config */) const { return true; /* always reload */ } + /// Updates the object from the configuration without reloading as much as possible. virtual void updateObjectFromConfigWithoutReloading( IExternalLoadable & /* object */, const Poco::Util::AbstractConfiguration & /* config */, const String & /* key_in_config */) const {} From 0240ffa5a6dab4f747d17295648d030af907f8f6 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 11 Mar 2024 19:31:52 +0100 Subject: [PATCH 0172/1165] Fetch the changed files only once --- tests/ci/pr_info.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 6f4b400f7a8..9bd30f3c58e 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -337,6 +337,9 @@ class PRInfo: ) def fetch_changed_files(self): + if self.changed_files_requested: + return + if not getattr(self, "diff_urls", False): raise TypeError("The event does not have diff URLs") From 49017fc163174e4cb4b3e9253094f399503add42 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 11 Mar 2024 20:39:47 +0100 Subject: [PATCH 0173/1165] Run the style-check on top of the merge-commit of PRs --- tests/ci/style_check.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 7fb5d22a3ce..4daa6921d6b 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -67,26 +67,6 @@ def parse_args(): return parser.parse_args() -def checkout_head(pr_info: PRInfo) -> None: - # It works ONLY for PRs, and only over ssh, so either - # ROBOT_CLICKHOUSE_SSH_KEY should be set or ssh-agent should work - assert pr_info.number - if not pr_info.head_name == pr_info.base_name: - # We can't push to forks, sorry folks - return - remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"] - fetch_cmd = ( - f"{GIT_PREFIX} fetch --depth=1 " - f"{remote_url} {pr_info.head_ref}:head-{pr_info.head_ref}" - ) - if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""): - with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): - git_runner(fetch_cmd) - else: - git_runner(fetch_cmd) - git_runner(f"git checkout -f head-{pr_info.head_ref}") - - def commit_push_staged(pr_info: PRInfo) -> None: # It works ONLY for PRs, and only over ssh, so either # ROBOT_CLICKHOUSE_SSH_KEY should be set or ssh-agent should work @@ -177,8 +157,6 @@ def main(): _ = future1.result() if run_pycheck: - if args.push: - checkout_head(pr_info) logging.info("Run py files check: %s", cmd_py) future2 = executor.submit(subprocess.run, cmd_py, shell=True) _ = future2.result() From c3f386d06a2c12f08d168accbf51457ad7a735c0 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 11 Mar 2024 20:57:35 +0100 Subject: [PATCH 0174/1165] Fix the autofixer --- tests/ci/style_check.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 4daa6921d6b..0332383df91 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -78,24 +78,23 @@ def commit_push_staged(pr_info: PRInfo) -> None: if not git_staged: return remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"] + head = git_runner("git rev-parse HEAD^{}") git_runner(f"{GIT_PREFIX} commit -m 'Automatic style fix'") - push_cmd = ( - f"{GIT_PREFIX} push {remote_url} head-{pr_info.head_ref}:{pr_info.head_ref}" + # The fetch to avoid issue 'pushed branch tip is behind its remote' + fetch_cmd = ( + f"{GIT_PREFIX} fetch {remote_url} --no-recurse-submodules --depth=2 {head}" ) + push_cmd = f"{GIT_PREFIX} push {remote_url} HEAD:{pr_info.head_ref}" if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""): with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): + git_runner(fetch_cmd) git_runner(push_cmd) - else: - git_runner(push_cmd) + return + + git_runner(fetch_cmd) + git_runner(push_cmd) -def checkout_last_ref(pr_info: PRInfo) -> None: - # Checkout the merge commit back to avoid special effects - assert pr_info.number - if not pr_info.head_name == pr_info.base_name: - # We can't push to forks, sorry folks - return - git_runner("git checkout -f -") def main(): @@ -162,7 +161,6 @@ def main(): _ = future2.result() if args.push: commit_push_staged(pr_info) - checkout_last_ref(pr_info) subprocess.check_call( f"python3 ../../utils/check-style/process_style_check_result.py --in-results-dir {temp_path} " From 41432c56a8f13a8914b65b265d9855847314db79 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 12 Mar 2024 10:50:44 +0100 Subject: [PATCH 0175/1165] Enable shellcheck conditionally --- tests/ci/style_check.py | 84 +++++++++++-------- utils/check-style/check_shell.sh | 13 +++ .../check-style/process_style_check_result.py | 4 +- 3 files changed, 64 insertions(+), 37 deletions(-) create mode 100755 utils/check-style/check_shell.sh diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 0332383df91..797c0ef12d0 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -8,8 +8,9 @@ import subprocess import sys from concurrent.futures import ProcessPoolExecutor from pathlib import Path -from typing import List, Tuple +from typing import List, Tuple, Union +import magic from docker_images_helper import get_docker_image, pull_image from env_helper import CI, REPO_COPY, TEMP_PATH from git_helper import GIT_PREFIX, git_runner @@ -95,6 +96,24 @@ def commit_push_staged(pr_info: PRInfo) -> None: git_runner(push_cmd) +def is_python(file: Union[Path, str]) -> bool: + """returns if the changed file in the repository is python script""" + # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, + # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 + return bool( + magic.from_file(os.path.join(REPO_COPY, file), mime=True) + == "text/x-script.python" + ) + + +def is_shell(file: Union[Path, str]) -> bool: + """returns if the changed file in the repository is shell script""" + # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, + # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 + return bool( + magic.from_file(os.path.join(REPO_COPY, file), mime=True) + == "text/x-shellscript" + ) def main(): @@ -111,29 +130,28 @@ def main(): temp_path.mkdir(parents=True, exist_ok=True) pr_info = PRInfo() + run_cpp_check = True + run_shell_check = True + run_python_check = True + if CI and pr_info.number > 0: + pr_info.fetch_changed_files() + run_cpp_check = not any( + is_python(file) or is_shell(file) for file in pr_info.changed_files + ) + run_shell_check = any(is_shell(file) for file in pr_info.changed_files) + run_python_check = any(is_python(file) for file in pr_info.changed_files) IMAGE_NAME = "clickhouse/style-test" image = pull_image(get_docker_image(IMAGE_NAME)) - cmd_cpp = ( + docker_command = ( f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output " - f"--entrypoint= -w/ClickHouse/utils/check-style " - f"{image} ./check_cpp.sh" - ) - - cmd_py = ( - f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " - f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output " - f"--entrypoint= -w/ClickHouse/utils/check-style " - f"{image} ./check_py.sh" - ) - - cmd_docs = ( - f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " - f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output " - f"--entrypoint= -w/ClickHouse/utils/check-style " - f"{image} ./check_docs.sh" + f"--entrypoint= -w/ClickHouse/utils/check-style {image}" ) + cmd_docs = f"{docker_command} ./check_docs.sh" + cmd_cpp = f"{docker_command} ./check_cpp.sh" + cmd_py = f"{docker_command} ./check_py.sh" + cmd_shell = f"{docker_command} ./check_shell.sh" with ProcessPoolExecutor(max_workers=2) as executor: logging.info("Run docs files check: %s", cmd_docs) @@ -141,26 +159,22 @@ def main(): # Parallelization does not make it faster - run subsequently _ = future.result() - run_cppcheck = True - run_pycheck = True - if CI and pr_info.number > 0: - pr_info.fetch_changed_files() - if not any(file.endswith(".py") for file in pr_info.changed_files): - run_pycheck = False - if all(file.endswith(".py") for file in pr_info.changed_files): - run_cppcheck = False - - if run_cppcheck: + if run_cpp_check: logging.info("Run source files check: %s", cmd_cpp) - future1 = executor.submit(subprocess.run, cmd_cpp, shell=True) - _ = future1.result() + future = executor.submit(subprocess.run, cmd_cpp, shell=True) + _ = future.result() - if run_pycheck: + if run_python_check: logging.info("Run py files check: %s", cmd_py) - future2 = executor.submit(subprocess.run, cmd_py, shell=True) - _ = future2.result() - if args.push: - commit_push_staged(pr_info) + future = executor.submit(subprocess.run, cmd_py, shell=True) + _ = future.result() + if run_shell_check: + logging.info("Run shellcheck check: %s", cmd_shell) + future = executor.submit(subprocess.run, cmd_shell, shell=True) + _ = future.result() + + if args.push: + commit_push_staged(pr_info) subprocess.check_call( f"python3 ../../utils/check-style/process_style_check_result.py --in-results-dir {temp_path} " diff --git a/utils/check-style/check_shell.sh b/utils/check-style/check_shell.sh new file mode 100755 index 00000000000..94f06220f88 --- /dev/null +++ b/utils/check-style/check_shell.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv + +start_total=$(date +%s) + +start=$(date +%s) +./shellcheck-run.sh |& tee /test_output/shellcheck.txt +runtime=$(($(date +%s)-start)) +echo "Check shellcheck. Done. $runtime seconds." + +runtime=$(($(date +%s)-start_total)) +echo "Check style total. Done. $runtime seconds." diff --git a/utils/check-style/process_style_check_result.py b/utils/check-style/process_style_check_result.py index 8c9837b4fc6..f1ca53eba3b 100755 --- a/utils/check-style/process_style_check_result.py +++ b/utils/check-style/process_style_check_result.py @@ -13,8 +13,8 @@ def process_result(result_folder): description = "" test_results = [] checks = ( - # "duplicate includes", - # "shellcheck", + "duplicate includes", + "shellcheck", "style", "pylint", "black", From 0e6b5705fbdce4c7032d39dbdae452f266e1b0b3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 12 Mar 2024 12:20:30 +0100 Subject: [PATCH 0176/1165] Install python-magic in style-check, split pip and apt --- docker/test/style/Dockerfile | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index b4ffcfb597c..5d53d03606f 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -8,20 +8,22 @@ ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ - aspell \ - curl \ - git \ - file \ - libxml2-utils \ - moreutils \ - python3-fuzzywuzzy \ - python3-pip \ - yamllint \ - locales \ - && pip3 install black==23.12.0 boto3 codespell==2.2.1 mypy==1.8.0 PyGithub unidiff pylint==3.1.0 \ - requests types-requests \ + aspell \ + curl \ + git \ + file \ + libxml2-utils \ + moreutils \ + python3-fuzzywuzzy \ + python3-pip \ + yamllint \ + locales \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* + +# python-magic is the same version as in Ubuntu 22.04 +RUN pip3 install black==23.12.0 boto3 codespell==2.2.1 mypy==1.8.0 PyGithub unidiff pylint==3.1.0 \ + python-magic==0.4.24 requests types-requests \ && rm -rf /root/.cache/pip RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8 From 97b0513f9928de5f0207978d8138b478bc1677bd Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 12 Mar 2024 19:21:05 +0100 Subject: [PATCH 0177/1165] Enable duplicate includes back, rewrite and parallelize them --- utils/check-style/check-duplicate-includes.sh | 5 ++- utils/check-style/check_cpp.sh | 31 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/utils/check-style/check-duplicate-includes.sh b/utils/check-style/check-duplicate-includes.sh index 64aca4d180d..ca8b98d9537 100755 --- a/utils/check-style/check-duplicate-includes.sh +++ b/utils/check-style/check-duplicate-includes.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash ROOT_PATH=$(git rev-parse --show-toplevel) +NPROC=$(($(nproc) + 3)) # Find duplicate include directives -find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | while read file; do grep -P '^#include ' $file | sort | uniq -c | grep -v -P '^\s+1\s' && echo $file; done | sed '/^[[:space:]]*$/d' +find "$ROOT_PATH"/{src,base,programs,utils} -type f '(' -name '*.h' -or -name '*.cpp' ')' -print0 | \ + xargs -0 -I {} -P "$NPROC" bash -c 'grep -P "^#include " "{}" | sort | uniq -c | grep -v -P "^\s+1\s" && echo "{}" '| \ + sed '/^\s*$/d' diff --git a/utils/check-style/check_cpp.sh b/utils/check-style/check_cpp.sh index ea90d79418c..dc1dae70e22 100755 --- a/utils/check-style/check_cpp.sh +++ b/utils/check-style/check_cpp.sh @@ -4,35 +4,32 @@ cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv -start_total=`date +%s` +start_total=$(date +%s) -# FIXME: 30 sec to wait -# echo "Check duplicates" | ts -# ./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt +start=$(date +%s) +./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt +runtime=$(($(date +%s)-start)) +echo "Check duplicates. Done. $runtime seconds." -start=`date +%s` +start=$(date +%s) ./check-style -n |& tee /test_output/style_output.txt -runtime=$((`date +%s`-start)) +runtime=$(($(date +%s)-start)) echo "Check style. Done. $runtime seconds." -start=`date +%s` +start=$(date +%s) ./check-whitespaces -n |& tee /test_output/whitespaces_output.txt -runtime=$((`date +%s`-start)) +runtime=$(($(date +%s)-start)) echo "Check whitespaces. Done. $runtime seconds." -start=`date +%s` +start=$(date +%s) ./check-workflows |& tee /test_output/workflows_output.txt -runtime=$((`date +%s`-start)) +runtime=$(($(date +%s)-start)) echo "Check workflows. Done. $runtime seconds." -start=`date +%s` +start=$(date +%s) ./check-submodules |& tee /test_output/submodules_output.txt -runtime=$((`date +%s`-start)) +runtime=$(($(date +%s)-start)) echo "Check submodules. Done. $runtime seconds." -# FIXME: 6 min to wait -# echo "Check shell scripts with shellcheck" | ts -# ./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt - -runtime=$((`date +%s`-start_total)) +runtime=$(($(date +%s)-start_total)) echo "Check style total. Done. $runtime seconds." From a5331ea54462c5cdb300c7f48d28d8795fac1bc9 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 14 Mar 2024 13:49:05 +0100 Subject: [PATCH 0178/1165] Rewrite list-licenses.sh to use array+find --- utils/list-licenses/list-licenses.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils/list-licenses/list-licenses.sh b/utils/list-licenses/list-licenses.sh index f09168a0596..cc730464e8e 100755 --- a/utils/list-licenses/list-licenses.sh +++ b/utils/list-licenses/list-licenses.sh @@ -5,15 +5,16 @@ if [[ "$OSTYPE" == "darwin"* ]]; then GREP_CMD=ggrep FIND_CMD=gfind else - FIND_CMD=find - GREP_CMD=grep + FIND_CMD='find' + GREP_CMD='grep' fi ROOT_PATH="$(git rev-parse --show-toplevel)" LIBS_PATH="${ROOT_PATH}/contrib" -ls -1 -d ${LIBS_PATH}/*/ "${ROOT_PATH}/base/poco" | ${GREP_CMD} -F -v -- '-cmake' | LC_ALL=C sort | while read LIB; do - LIB_NAME=$(basename $LIB) +mapfile -t libs < <(echo "${ROOT_PATH}/base/poco"; find "${LIBS_PATH}" -type d -maxdepth 1 ! -name '*-cmake' | LC_ALL=C sort) +for LIB in "${libs[@]}"; do + LIB_NAME=$(basename "$LIB") LIB_LICENSE=$( LC_ALL=C ${FIND_CMD} "$LIB" -type f -and '(' -iname 'LICENSE*' -or -iname 'COPYING*' -or -iname 'COPYRIGHT*' ')' -and -not '(' -iname '*.html' -or -iname '*.htm' -or -iname '*.rtf' -or -name '*.cpp' -or -name '*.h' -or -iname '*.json' ')' -printf "%d\t%p\n" | From 070f7bee9b550962cba6768ffd3a74306e4e2107 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 14 Mar 2024 13:56:43 +0100 Subject: [PATCH 0179/1165] Fix shellcheck issues in durability-test.sh --- utils/durability-test/durability-test.sh | 51 +++++++++++------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/utils/durability-test/durability-test.sh b/utils/durability-test/durability-test.sh index 97c39473b69..188ef3452e7 100755 --- a/utils/durability-test/durability-test.sh +++ b/utils/durability-test/durability-test.sh @@ -15,7 +15,6 @@ Usage: URL=http://cloud-images.ubuntu.com/bionic/current IMAGE=bionic-server-cloudimg-amd64.img SSH_PORT=11022 -CLICKHOUSE_PORT=9090 PASSWORD=root TABLE_NAME=$1 @@ -34,19 +33,18 @@ function run() function copy() { - sshpass -p $PASSWORD scp -r -P $SSH_PORT $1 root@localhost:$2 2>/dev/null + sshpass -p $PASSWORD scp -r -P $SSH_PORT "$1" "root@localhost:$2" 2>/dev/null } function wait_vm_for_start() { echo "Waiting until VM started..." started=0 - for i in {0..100}; do - run "exit" - if [ $? -eq 0 ]; then + for _ in {0..100}; do + if run "exit"; then started=1 break - fi + fi sleep 1s done @@ -62,9 +60,8 @@ function wait_clickhouse_for_start() { echo "Waiting until ClickHouse started..." started=0 - for i in {0..30}; do - run "clickhouse client --query 'select 1'" > /dev/null - if [ $? -eq 0 ]; then + for _ in {0..30}; do + if run "clickhouse client --query 'select 1'" > /dev/null; then started=1 break fi @@ -105,11 +102,11 @@ if [[ -z $CLICKHOUSE_CONFIG_DIR ]]; then CLICKHOUSE_CONFIG_DIR=/etc/clickhouse-server fi -echo "Using ClickHouse binary:" $CLICKHOUSE_BINARY -echo "Using ClickHouse config from:" $CLICKHOUSE_CONFIG_DIR +echo "Using ClickHouse binary: $CLICKHOUSE_BINARY" +echo "Using ClickHouse config from: $CLICKHOUSE_CONFIG_DIR" -copy $CLICKHOUSE_BINARY /usr/bin -copy $CLICKHOUSE_CONFIG_DIR /etc +copy "$CLICKHOUSE_BINARY" /usr/bin +copy "$CLICKHOUSE_CONFIG_DIR" /etc run "mv /etc/$CLICKHOUSE_CONFIG_DIR /etc/clickhouse-server" echo "Prepared VM" @@ -118,23 +115,23 @@ echo "Starting ClickHouse" run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" & wait_clickhouse_for_start -query=`cat $CREATE_QUERY` -echo "Executing query:" $query +query=$(cat "$CREATE_QUERY") +echo "Executing query: $query" run "clickhouse client --query '$query'" -query=`cat $INSERT_QUERY` -echo "Will run in a loop query: " $query +query=$(cat "$INSERT_QUERY") +echo "Will run in a loop query: $query" run "clickhouse benchmark <<< '$query' -c 8" & echo "Running queries" -pid=`pidof qemu-system-x86_64` +pid=$(pidof qemu-system-x86_64) sec=$(( (RANDOM % 5) + 25 )) ms=$(( RANDOM % 1000 )) echo "Will kill VM in $sec.$ms sec" sleep $sec.$ms -kill -9 $pid +kill -9 "$pid" echo "Restarting" @@ -147,22 +144,22 @@ run "rm -r *data/system" run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" & wait_clickhouse_for_start -pid=`pidof qemu-system-x86_64` -result=`run "grep $TABLE_NAME clickhouse-server.log | grep 'Caught exception while loading metadata'"` +pid=$(pidof qemu-system-x86_64) +result=$(run "grep $TABLE_NAME clickhouse-server.log | grep 'Caught exception while loading metadata'") if [[ -n $result ]]; then echo "FAIL. Can't attach table:" - echo $result - kill -9 $pid + echo "$result" + kill -9 "$pid" exit 1 fi -result=`run "grep $TABLE_NAME clickhouse-server.log | grep 'Considering to remove broken part'"` +result=$(run "grep $TABLE_NAME clickhouse-server.log | grep 'Considering to remove broken part'") if [[ -n $result ]]; then echo "FAIL. Have broken parts:" - echo $result - kill -9 $pid + echo "$result" + kill -9 "$pid" exit 1 fi -kill -9 $pid +kill -9 "$pid" echo OK From 62c91fdddc8c0da90b8ef14ecd5a0f8c442073d3 Mon Sep 17 00:00:00 2001 From: Zhuo Qiu Date: Mon, 18 Mar 2024 22:33:07 +0800 Subject: [PATCH 0180/1165] revise docs Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> --- docs/en/operations/settings/merge-tree-settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 9ce705ff805..3e411a51ff4 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -884,7 +884,7 @@ Default value: false ## load_existing_rows_count_for_old_parts {#load_existing_rows_count_for_old_parts} -If enabled along with [exclude_deleted_rows_for_part_size_in_merge](#exclude_deleted_rows_for_part_size_in_merge), deleted rows count for existing data parts will be calculated during table starting up. +If enabled along with [exclude_deleted_rows_for_part_size_in_merge](#exclude_deleted_rows_for_part_size_in_merge), deleted rows count for existing data parts will be calculated during table starting up. Note that it may slow down start up table loading. Possible values: From 0530055842a14468218d774d020baf0153abfea7 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 18 Mar 2024 15:59:05 +0100 Subject: [PATCH 0181/1165] Updated names in BufferAllocationPolicy --- src/Common/BufferAllocationPolicy.cpp | 6 +-- src/Common/BufferAllocationPolicy.h | 41 ++++++++++--------- .../IO/WriteBufferFromAzureBlobStorage.cpp | 12 +++--- .../IO/WriteBufferFromAzureBlobStorage.h | 2 +- src/IO/WriteBufferFromS3.cpp | 12 +++--- src/IO/WriteBufferFromS3.h | 2 +- 6 files changed, 39 insertions(+), 36 deletions(-) diff --git a/src/Common/BufferAllocationPolicy.cpp b/src/Common/BufferAllocationPolicy.cpp index 1456233eb03..2b00d4f84d0 100644 --- a/src/Common/BufferAllocationPolicy.cpp +++ b/src/Common/BufferAllocationPolicy.cpp @@ -11,7 +11,7 @@ class FixedSizeBufferAllocationPolicy : public IBufferAllocationPolicy size_t buffer_number = 0; public: - explicit FixedSizeBufferAllocationPolicy(const BufferAllocationSettings & settings_) + explicit FixedSizeBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_) : buffer_size(settings_.strict_upload_part_size) { chassert(buffer_size > 0); @@ -45,7 +45,7 @@ class ExpBufferAllocationPolicy : public DB::IBufferAllocationPolicy size_t buffer_number = 0; public: - explicit ExpBufferAllocationPolicy(const BufferAllocationSettings & settings_) + explicit ExpBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_) : first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size)) , second_size(settings_.min_upload_part_size) , multiply_factor(settings_.upload_part_size_multiply_factor) @@ -91,7 +91,7 @@ public: IBufferAllocationPolicy::~IBufferAllocationPolicy() = default; -IBufferAllocationPolicyPtr ChooseBufferPolicy(BufferAllocationSettings settings_) +IBufferAllocationPolicy::IBufferAllocationPolicyPtr IBufferAllocationPolicy::create(IBufferAllocationPolicy::Settings settings_) { if (settings_.strict_upload_part_size > 0) return std::make_unique(settings_); diff --git a/src/Common/BufferAllocationPolicy.h b/src/Common/BufferAllocationPolicy.h index b759d22ede6..3ada9e52fdf 100644 --- a/src/Common/BufferAllocationPolicy.h +++ b/src/Common/BufferAllocationPolicy.h @@ -9,27 +9,30 @@ namespace DB { -struct BufferAllocationSettings -{ - size_t strict_upload_part_size = 0; - size_t min_upload_part_size = 16 * 1024 * 1024; - size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; - size_t upload_part_size_multiply_factor = 2; - size_t upload_part_size_multiply_parts_count_threshold = 500; - size_t max_single_part_upload_size = 32 * 1024 * 1024; -}; - class IBufferAllocationPolicy { - public: - virtual size_t getBufferNumber() const = 0; - virtual size_t getBufferSize() const = 0; - virtual void nextBuffer() = 0; - virtual ~IBufferAllocationPolicy() = 0; +public: + + struct Settings + { + size_t strict_upload_part_size = 0; + size_t min_upload_part_size = 16 * 1024 * 1024; + size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; + size_t upload_part_size_multiply_factor = 2; + size_t upload_part_size_multiply_parts_count_threshold = 500; + size_t max_single_part_upload_size = 32 * 1024 * 1024; + }; + + + virtual size_t getBufferNumber() const = 0; + virtual size_t getBufferSize() const = 0; + virtual void nextBuffer() = 0; + virtual ~IBufferAllocationPolicy() = 0; + + using IBufferAllocationPolicyPtr = std::unique_ptr; + + static IBufferAllocationPolicyPtr create(Settings settings_); + }; -using IBufferAllocationPolicyPtr = std::unique_ptr; - -IBufferAllocationPolicyPtr ChooseBufferPolicy(BufferAllocationSettings settings_); - } diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index bc11d445a51..41d7187afda 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -33,12 +33,12 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( ThreadPoolCallbackRunner schedule_) : WriteBufferFromFileBase(buf_size_, nullptr, 0) , log(getLogger("WriteBufferFromAzureBlobStorage")) - , buffer_allocation_policy(ChooseBufferPolicy({settings_->strict_upload_part_size, - settings_->min_upload_part_size, - settings_->max_upload_part_size, - settings_->upload_part_size_multiply_factor, - settings_->upload_part_size_multiply_parts_count_threshold, - settings_->max_single_part_upload_size})) + , buffer_allocation_policy(IBufferAllocationPolicy::create({settings_->strict_upload_part_size, + settings_->min_upload_part_size, + settings_->max_upload_part_size, + settings_->upload_part_size_multiply_factor, + settings_->upload_part_size_multiply_parts_count_threshold, + settings_->max_single_part_upload_size})) , max_single_part_upload_size(settings_->max_single_part_upload_size) , max_unexpected_write_error_retries(settings_->max_unexpected_write_error_retries) , blob_path(blob_path_) diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index 7223f66693e..f047dd53724 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -62,7 +62,7 @@ private: LoggerPtr log; LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); - IBufferAllocationPolicyPtr buffer_allocation_policy; + IBufferAllocationPolicy::IBufferAllocationPolicyPtr buffer_allocation_policy; const size_t max_single_part_upload_size; const size_t max_unexpected_write_error_retries; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 60fa828d6c4..c907c4cb6b6 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -93,12 +93,12 @@ WriteBufferFromS3::WriteBufferFromS3( , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) , object_metadata(std::move(object_metadata_)) - , buffer_allocation_policy(ChooseBufferPolicy({upload_settings.strict_upload_part_size, - upload_settings.min_upload_part_size, - upload_settings.max_upload_part_size, - upload_settings.upload_part_size_multiply_factor, - upload_settings.upload_part_size_multiply_parts_count_threshold, - upload_settings.max_single_part_upload_size})) + , buffer_allocation_policy(IBufferAllocationPolicy::create({upload_settings.strict_upload_part_size, + upload_settings.min_upload_part_size, + upload_settings.max_upload_part_size, + upload_settings.upload_part_size_multiply_factor, + upload_settings.upload_part_size_multiply_parts_count_threshold, + upload_settings.max_single_part_upload_size})) , task_tracker( std::make_unique( std::move(schedule_), diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 840274c8ace..201743e7f9c 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -86,7 +86,7 @@ private: LoggerPtr log = getLogger("WriteBufferFromS3"); LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); - IBufferAllocationPolicyPtr buffer_allocation_policy; + IBufferAllocationPolicy::IBufferAllocationPolicyPtr buffer_allocation_policy; /// Upload in S3 is made in parts. /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts. From 679bf92cd146f0855ba5e02bc0c343c46506b534 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 18 Mar 2024 15:08:46 +0000 Subject: [PATCH 0182/1165] Split test 00159_parallel_formatting_json_and_friends to avoid timeouts --- ...l_formatting_json_and_friends_1.reference} | 12 ----------- ...parallel_formatting_json_and_friends_1.sh} | 2 +- ...el_formatting_json_and_friends_2.reference | 12 +++++++++++ ..._parallel_formatting_json_and_friends_2.sh | 20 +++++++++++++++++++ 4 files changed, 33 insertions(+), 13 deletions(-) rename tests/queries/1_stateful/{00159_parallel_formatting_json_and_friends.reference => 00159_parallel_formatting_json_and_friends_1.reference} (51%) rename tests/queries/1_stateful/{00159_parallel_formatting_json_and_friends.sh => 00159_parallel_formatting_json_and_friends_1.sh} (83%) create mode 100644 tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.reference create mode 100755 tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.sh diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.reference similarity index 51% rename from tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference rename to tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.reference index 787b57ce761..cd425198651 100644 --- a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.reference @@ -14,15 +14,3 @@ JSONCompactEachRowWithNames, false e3231b1c8187de4da6752d692b2ddba9 - JSONCompactEachRowWithNames, true e3231b1c8187de4da6752d692b2ddba9 - -JSONCompactStringsEachRowWithNames, false -e3231b1c8187de4da6752d692b2ddba9 - -JSONCompactStringsEachRowWithNames, true -e3231b1c8187de4da6752d692b2ddba9 - -JSONCompactEachRowWithNamesAndTypes, false -d40c4327c63eded184eee185a5330e12 - -JSONCompactEachRowWithNamesAndTypes, true -d40c4327c63eded184eee185a5330e12 - -JSONCompactStringsEachRowWithNamesAndTypes, false -d40c4327c63eded184eee185a5330e12 - -JSONCompactStringsEachRowWithNamesAndTypes, true -d40c4327c63eded184eee185a5330e12 - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.sh similarity index 83% rename from tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh rename to tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.sh index 77dd330099a..0cc4636ef5d 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -FORMATS=('JSONEachRow' 'JSONCompactEachRow' 'JSONCompactStringsEachRow' 'JSONCompactEachRowWithNames' 'JSONCompactStringsEachRowWithNames' 'JSONCompactEachRowWithNamesAndTypes' 'JSONCompactStringsEachRowWithNamesAndTypes') +FORMATS=('JSONEachRow' 'JSONCompactEachRow' 'JSONCompactStringsEachRow' 'JSONCompactEachRowWithNames') for format in "${FORMATS[@]}" do diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.reference b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.reference new file mode 100644 index 00000000000..ba7a8ac5fdc --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.reference @@ -0,0 +1,12 @@ +JSONCompactStringsEachRowWithNames, false +e3231b1c8187de4da6752d692b2ddba9 - +JSONCompactStringsEachRowWithNames, true +e3231b1c8187de4da6752d692b2ddba9 - +JSONCompactEachRowWithNamesAndTypes, false +d40c4327c63eded184eee185a5330e12 - +JSONCompactEachRowWithNamesAndTypes, true +d40c4327c63eded184eee185a5330e12 - +JSONCompactStringsEachRowWithNamesAndTypes, false +d40c4327c63eded184eee185a5330e12 - +JSONCompactStringsEachRowWithNamesAndTypes, true +d40c4327c63eded184eee185a5330e12 - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.sh b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.sh new file mode 100755 index 00000000000..1c49305fe53 --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +FORMATS=('JSONCompactStringsEachRowWithNames' 'JSONCompactEachRowWithNamesAndTypes' 'JSONCompactStringsEachRowWithNamesAndTypes') + +for format in "${FORMATS[@]}" +do + echo "$format, false"; + $CLICKHOUSE_CLIENT --max_threads=0 --output_format_parallel_formatting=false -q \ + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum + + echo "$format, true"; + $CLICKHOUSE_CLIENT --max_threads=0 --output_format_parallel_formatting=true -q \ + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum +done From 514faa851c32d26620fb3a086f3dbfe40ecd1a23 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 18 Mar 2024 15:26:01 +0000 Subject: [PATCH 0183/1165] Make test 00159_parallel_formatting_tsv_and_friends.sh faster --- .../1_stateful/00159_parallel_formatting_tsv_and_friends.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh index 02441190b91..393a9fc4282 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh @@ -11,9 +11,9 @@ for format in "${FORMATS[@]}" do echo "$format, false"; $CLICKHOUSE_CLIENT --max_threads=0 --output_format_parallel_formatting=false -q \ - "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum echo "$format, true"; $CLICKHOUSE_CLIENT --max_threads=0 --output_format_parallel_formatting=true -q \ - "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum done From caaaacb9a22eb6f0d35616df268c1682936f0838 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 18 Mar 2024 16:21:46 +0100 Subject: [PATCH 0184/1165] Fix case when only one of the limits (size || elements_count) is not satisfied, which lead to it not being satisfied at second phase of space reservation --- src/Interpreters/Cache/FileCache.cpp | 62 ++++++++------- src/Interpreters/Cache/FileCache.h | 3 + src/Interpreters/Cache/IFileCachePriority.h | 28 ++++++- .../Cache/LRUFileCachePriority.cpp | 42 +++++++++-- src/Interpreters/Cache/LRUFileCachePriority.h | 15 +++- .../Cache/SLRUFileCachePriority.cpp | 75 +++++++++++++++---- .../Cache/SLRUFileCachePriority.h | 6 ++ 7 files changed, 183 insertions(+), 48 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index bc8236f37fb..7d6df156cc0 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -823,10 +823,13 @@ bool FileCache::tryReserve( } EvictionCandidates eviction_candidates; + bool reached_size_limit = false; + bool reached_elements_limit = false; if (query_priority) { - if (!query_priority->collectCandidatesForEviction(size, reserve_stat, eviction_candidates, {}, user.user_id, cache_lock)) + if (!query_priority->collectCandidatesForEviction( + size, reserve_stat, eviction_candidates, {}, user.user_id, reached_size_limit, reached_elements_limit, cache_lock)) return false; LOG_TEST(log, "Query limits satisfied (while reserving for {}:{})", file_segment.key(), file_segment.offset()); @@ -839,38 +842,41 @@ bool FileCache::tryReserve( auto queue_iterator = file_segment.getQueueIterator(); chassert(!queue_iterator || file_segment.getReservedSize() > 0); - if (!main_priority->collectCandidatesForEviction(size, reserve_stat, eviction_candidates, queue_iterator, user.user_id, cache_lock)) + if (!main_priority->collectCandidatesForEviction( + size, reserve_stat, eviction_candidates, queue_iterator, user.user_id, reached_size_limit, reached_elements_limit, cache_lock)) return false; - /// Let's release cache lock if we are going to remove files from filesystem. - const bool release_lock = eviction_candidates.size() > 0; - if (release_lock) - cache_lock.unlock(); - if (!file_segment.getKeyMetadata()->createBaseDirectory()) return false; - try + if (eviction_candidates.size() > 0) { - /// Remove eviction candidates from filesystem. - eviction_candidates.evict(); - } - catch (...) - { - if (release_lock) + chassert(reached_size_limit || reached_elements_limit); + + std::unique_ptr hold_space; + if (!reached_size_limit) + hold_space = std::make_unique(size, 0, queue_iterator, *main_priority, cache_lock); + else if (!reached_elements_limit) + hold_space = std::make_unique(0, 1, queue_iterator, *main_priority, cache_lock); + + cache_lock.unlock(); + try + { + /// Remove eviction candidates from filesystem. + eviction_candidates.evict(); + } + catch (...) + { cache_lock.lock(); + /// Invalidate queue entries if some succeeded to be removed. + eviction_candidates.finalize(query_context.get(), cache_lock); + throw; + } - /// Invalidate queue entries if some succeeded to be removed. - eviction_candidates.finalize(query_context.get(), cache_lock); - throw; - } - - /// Take cache lock again. - if (release_lock) cache_lock.lock(); - - /// Invalidate and remove queue entries and execute (only for SLRU) finalize func. - eviction_candidates.finalize(query_context.get(), cache_lock); + /// Invalidate and remove queue entries and execute (only for SLRU) finalize func. + eviction_candidates.finalize(query_context.get(), cache_lock); + } /// Space reservation is incremental, so file_segment_metadata is created first (with state Empty), /// and queue_iterator is assigned on first space reservation attempt @@ -887,9 +893,6 @@ bool FileCache::tryReserve( file_segment.setQueueIterator(queue_iterator); } - file_segment.reserved_size += size; - chassert(file_segment.reserved_size == queue_iterator->getEntry()->size); - if (query_context) { auto query_queue_it = query_context->tryGet(file_segment.key(), file_segment.offset(), cache_lock); @@ -899,6 +902,9 @@ bool FileCache::tryReserve( query_context->add(file_segment.getKeyMetadata(), file_segment.offset(), size, user, cache_lock); } + file_segment.reserved_size += size; + chassert(file_segment.reserved_size == queue_iterator->getEntry()->size); + if (main_priority->getSize(cache_lock) > (1ull << 63)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache became inconsistent. There must be a bug"); @@ -1161,7 +1167,7 @@ void FileCache::loadMetadataForKeys(const fs::path & keys_dir) auto lock = lockCache(); size_limit = main_priority->getSizeLimit(lock); - limits_satisfied = main_priority->canFit(size, lock, nullptr, true); + limits_satisfied = main_priority->canFit(size, 1, lock, nullptr, true); if (limits_satisfied) cache_it = main_priority->add(key_metadata, offset, size, user, lock, /* best_effort */true); diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 8ea5f4dab40..f1384916cd0 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -46,6 +46,9 @@ struct FileCacheReserveStat } }; + bool reached_elements_limit = false; + bool reached_size_limit = false; + Stat stat; std::unordered_map stat_by_kind; diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index 651710656af..e1fb96c9a7d 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -102,6 +102,7 @@ public: /// for the corresponding file segment. virtual bool canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock &, IteratorPtr reservee = nullptr, bool best_effort = false) const = 0; @@ -120,15 +121,40 @@ public: size_t size, FileCacheReserveStat & stat, EvictionCandidates & res, - IFileCachePriority::IteratorPtr reservee, + IteratorPtr reservee, const UserID & user_id, + bool & reached_size_limit, + bool & reached_elements_limit, const CachePriorityGuard::Lock &) = 0; virtual bool modifySizeLimits(size_t max_size_, size_t max_elements_, double size_ratio_, const CachePriorityGuard::Lock &) = 0; + struct HoldSpace : boost::noncopyable + { + HoldSpace(size_t size_, size_t elements_, IteratorPtr reservee_, IFileCachePriority & priority_, const CachePriorityGuard::Lock & lock) + : size(size_), elements(elements_), reservee(reservee_), priority(priority_) + { + priority.holdImpl(size, elements, reservee, lock); + } + + ~HoldSpace() + { + priority.releaseImpl(size, elements, reservee); + } + + size_t size; + size_t elements; + IteratorPtr reservee; + IFileCachePriority & priority; + }; + HoldSpace takeHold(); + protected: IFileCachePriority(size_t max_size_, size_t max_elements_); + virtual void holdImpl(size_t size, size_t elements, IteratorPtr reservee, const CachePriorityGuard::Lock & lock) = 0; + virtual void releaseImpl(size_t size, size_t elements, IteratorPtr) = 0; + size_t max_size = 0; size_t max_elements = 0; }; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 6285dc684cf..e1004cc5245 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -219,21 +219,32 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CachePriorityGuard bool LRUFileCachePriority::canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock & lock, IteratorPtr, bool) const { - return canFit(size, 0, 0, lock); + return canFit(size, elements, 0, 0, nullptr, nullptr, lock); } bool LRUFileCachePriority::canFit( size_t size, + size_t elements, size_t released_size_assumption, size_t released_elements_assumption, + bool * reached_size_limit, + bool * reached_elements_limit, const CachePriorityGuard::Lock &) const { - return (max_size == 0 || (state->current_size + size - released_size_assumption <= max_size)) - && (max_elements == 0 || state->current_elements_num + 1 - released_elements_assumption <= max_elements); + const bool size_limit_satisifed = max_size == 0 || state->current_size + size - released_size_assumption <= max_size; + const bool elements_limit_satisfied = max_elements == 0 || state->current_elements_num + elements - released_elements_assumption <= max_elements; + + if (reached_size_limit) + *reached_size_limit |= !size_limit_satisifed; + if (reached_elements_limit) + *reached_elements_limit |= !elements_limit_satisfied; + + return size_limit_satisifed && elements_limit_satisfied; } bool LRUFileCachePriority::collectCandidatesForEviction( @@ -242,9 +253,11 @@ bool LRUFileCachePriority::collectCandidatesForEviction( EvictionCandidates & res, IFileCachePriority::IteratorPtr, const UserID &, + bool & reached_size_limit, + bool & reached_elements_limit, const CachePriorityGuard::Lock & lock) { - if (canFit(size, lock)) + if (canFit(size, 1, 0, 0, &reached_size_limit, &reached_elements_limit, lock)) return true; ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictionTries); @@ -270,7 +283,7 @@ bool LRUFileCachePriority::collectCandidatesForEviction( auto can_fit = [&] { - return canFit(size, stat.stat.releasable_size, stat.stat.releasable_count, lock); + return canFit(size, 1, stat.stat.releasable_size, stat.stat.releasable_count, nullptr, nullptr, lock); }; iterate([&](LockedKey & locked_key, const FileSegmentMetadataPtr & segment_metadata) @@ -450,4 +463,23 @@ void LRUFileCachePriority::shuffle(const CachePriorityGuard::Lock &) queue.splice(queue.end(), queue, it); } +void LRUFileCachePriority::holdImpl(size_t size, size_t elements, IteratorPtr, const CachePriorityGuard::Lock & lock) +{ + if (!canFit(size, elements, lock)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot take space {}. Current state {}/{} in size, {}/{} in elements", + size, state->current_size, max_size, state->current_elements_num, max_elements); + } + + state->current_size += size; + state->current_elements_num += elements; +} + +void LRUFileCachePriority::releaseImpl(size_t size, size_t elements, IteratorPtr) +{ + state->current_size -= size; + state->current_elements_num -= elements; +} + } diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index d5151b6bbff..d292ff89556 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -30,6 +30,7 @@ public: bool canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock &, IteratorPtr reservee = nullptr, bool best_effort = false) const override; @@ -48,6 +49,8 @@ public: EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, const UserID & user_id, + bool & reached_size_limit, + bool & reached_elements_limit, const CachePriorityGuard::Lock &) override; void shuffle(const CachePriorityGuard::Lock &) override; @@ -76,7 +79,14 @@ private: void updateElementsCount(int64_t num); void updateSize(int64_t size); - bool canFit(size_t size, size_t released_size_assumption, size_t released_elements_assumption, const CachePriorityGuard::Lock &) const; + bool canFit( + size_t size, + size_t elements, + size_t released_size_assumption, + size_t released_elements_assumption, + bool * reached_size_limit, + bool * reached_elements_limit, + const CachePriorityGuard::Lock &) const; LRUQueue::iterator remove(LRUQueue::iterator it, const CachePriorityGuard::Lock &); @@ -91,6 +101,9 @@ private: LRUIterator move(LRUIterator & it, LRUFileCachePriority & other, const CachePriorityGuard::Lock &); LRUIterator add(EntryPtr entry, const CachePriorityGuard::Lock &); + + void holdImpl(size_t size, size_t elements, IteratorPtr, const CachePriorityGuard::Lock & lock) override; + void releaseImpl(size_t size, size_t elements, IteratorPtr) override; }; class LRUFileCachePriority::LRUIterator : public IFileCachePriority::Iterator diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 57c4a888025..0bc9826f581 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -10,6 +10,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { size_t getRatio(size_t total, double ratio) @@ -46,23 +51,24 @@ size_t SLRUFileCachePriority::getElementsCount(const CachePriorityGuard::Lock & bool SLRUFileCachePriority::canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock & lock, IteratorPtr reservee, bool best_effort) const { if (best_effort) - return probationary_queue.canFit(size, lock) || protected_queue.canFit(size, lock); + return probationary_queue.canFit(size, elements, lock) || protected_queue.canFit(size, elements, lock); if (reservee) { const auto * slru_iterator = assert_cast(reservee.get()); if (slru_iterator->is_protected) - return protected_queue.canFit(size, lock); + return protected_queue.canFit(size, elements, lock); else - return probationary_queue.canFit(size, lock); + return probationary_queue.canFit(size, elements, lock); } else - return probationary_queue.canFit(size, lock); + return probationary_queue.canFit(size, elements, lock); } IFileCachePriority::IteratorPtr SLRUFileCachePriority::add( /// NOLINT @@ -78,7 +84,7 @@ IFileCachePriority::IteratorPtr SLRUFileCachePriority::add( /// NOLINT /// If it is server startup, we put entries in any queue it will fit in, /// but with preference for probationary queue, /// because we do not know the distribution between queues after server restart. - if (probationary_queue.canFit(size, lock)) + if (probationary_queue.canFit(size, 1, lock)) { auto lru_iterator = probationary_queue.add(std::make_shared(key_metadata->key, offset, size, key_metadata), lock); return std::make_shared(this, std::move(lru_iterator), false); @@ -102,13 +108,16 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, const UserID & user_id, + bool & reached_size_limit, + bool & reached_elements_limit, const CachePriorityGuard::Lock & lock) { /// If `it` is nullptr, then it is the first space reservation attempt /// for a corresponding file segment, so it will be directly put into probationary queue. if (!reservee) { - return probationary_queue.collectCandidatesForEviction(size, stat, res, reservee, user_id, lock); + return probationary_queue.collectCandidatesForEviction( + size, stat, res, reservee, user_id, reached_size_limit, reached_elements_limit, lock); } /// If `it` not nullptr (e.g. is already in some queue), @@ -116,13 +125,14 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( /// (in order to know where we need to free space). if (!assert_cast(reservee.get())->is_protected) { - return probationary_queue.collectCandidatesForEviction(size, stat, res, reservee, user_id, lock); + return probationary_queue.collectCandidatesForEviction( + size, stat, res, reservee, user_id, reached_size_limit, reached_elements_limit, lock); } /// Entry is in protected queue. /// Check if we have enough space in protected queue to fit a new size of entry. /// `size` is the increment to the current entry.size we want to increase. - if (protected_queue.canFit(size, lock)) + if (protected_queue.canFit(size, 1, lock)) return true; /// If not enough space - we need to "downgrade" lowest priority entries from protected @@ -132,13 +142,17 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( auto downgrade_candidates = std::make_shared(); FileCacheReserveStat downgrade_stat; - if (!protected_queue.collectCandidatesForEviction(size, downgrade_stat, *downgrade_candidates, reservee, user_id, lock)) + if (!protected_queue.collectCandidatesForEviction( + size, downgrade_stat, *downgrade_candidates, reservee, user_id, reached_size_limit, reached_elements_limit, lock)) return false; const size_t size_to_downgrade = downgrade_stat.stat.releasable_size; - if (!probationary_queue.canFit(size_to_downgrade, lock) - && !probationary_queue.collectCandidatesForEviction(size_to_downgrade, stat, res, reservee, user_id, lock)) + bool reached_size_limit_noop; + bool reached_elements_limit_noop; + if (!probationary_queue.canFit(size_to_downgrade, 1, lock) + && !probationary_queue.collectCandidatesForEviction( + size_to_downgrade, stat, res, reservee, user_id, reached_size_limit_noop, reached_elements_limit_noop, lock)) return false; res.setFinalizeEvictionFunc([=, this](const CachePriorityGuard::Lock & lk) mutable @@ -185,7 +199,11 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach EvictionCandidates downgrade_candidates; FileCacheReserveStat downgrade_stat; - if (!protected_queue.collectCandidatesForEviction(size, downgrade_stat, downgrade_candidates, {}, "", lock)) + bool reached_size_limit_noop; + bool reached_elements_limit_noop; + + if (!protected_queue.collectCandidatesForEviction( + size, downgrade_stat, downgrade_candidates, {}, "", reached_size_limit_noop, reached_elements_limit_noop, lock)) { /// We cannot make space for entry to be moved to protected queue /// (not enough releasable file segments). @@ -208,7 +226,8 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach if (size_to_free) { - if (!probationary_queue.collectCandidatesForEviction(size_to_free, stat, eviction_candidates, {}, {}, lock)) + if (!probationary_queue.collectCandidatesForEviction( + size_to_free, stat, eviction_candidates, {}, {}, reached_size_limit_noop, reached_elements_limit_noop, lock)) { /// "downgrade" candidates cannot be moved to probationary queue, /// so entry cannot be moved to protected queue as well. @@ -321,4 +340,34 @@ void SLRUFileCachePriority::SLRUIterator::assertValid() const lru_iterator.assertValid(); } +void SLRUFileCachePriority::holdImpl(size_t size, size_t elements, IteratorPtr reservee, const CachePriorityGuard::Lock & lock) +{ + if (!canFit(size, elements, lock, reservee)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot hold space {}", size); + + if (reservee) + { + const auto * slru_iterator = assert_cast(reservee.get()); + if (slru_iterator->is_protected) + return protected_queue.holdImpl(size, elements, reservee, lock); + else + return probationary_queue.holdImpl(size, elements, reservee, lock); + } + else + return probationary_queue.holdImpl(size, elements, reservee, lock); +} + +void SLRUFileCachePriority::releaseImpl(size_t size, size_t elements, IteratorPtr reservee) +{ + if (reservee) + { + const auto * slru_iterator = assert_cast(reservee.get()); + if (slru_iterator->is_protected) + return protected_queue.releaseImpl(size, elements, reservee); + else + return probationary_queue.releaseImpl(size, elements, reservee); + } + else + return probationary_queue.releaseImpl(size, elements, reservee); +} } diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h index 7b8ae77cc8f..286a2e7b9fd 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.h +++ b/src/Interpreters/Cache/SLRUFileCachePriority.h @@ -27,6 +27,7 @@ public: bool canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock &, IteratorPtr reservee = nullptr, bool best_effort = false) const override; @@ -45,6 +46,8 @@ public: EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, const UserID & user_id, + bool & reached_size_limit, + bool & reached_elements_limit, const CachePriorityGuard::Lock &) override; void shuffle(const CachePriorityGuard::Lock &) override; @@ -60,6 +63,9 @@ private: LoggerPtr log = getLogger("SLRUFileCachePriority"); void increasePriority(SLRUIterator & iterator, const CachePriorityGuard::Lock & lock); + + void holdImpl(size_t size, size_t elements, IteratorPtr reservee, const CachePriorityGuard::Lock & lock) override; + void releaseImpl(size_t size, size_t elements, IteratorPtr reservee) override; }; class SLRUFileCachePriority::SLRUIterator : public IFileCachePriority::Iterator From 051103b0e0fef928cc41aafd00de7565b776dfd8 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 18 Mar 2024 15:39:21 +0000 Subject: [PATCH 0185/1165] Fix db iterator wait during async metrics collection --- src/Databases/DatabaseAtomic.cpp | 4 +- src/Databases/DatabaseAtomic.h | 2 +- src/Databases/DatabaseDictionary.cpp | 2 +- src/Databases/DatabaseDictionary.h | 2 +- src/Databases/DatabaseFilesystem.cpp | 2 +- src/Databases/DatabaseFilesystem.h | 2 +- src/Databases/DatabaseHDFS.h | 2 +- src/Databases/DatabaseLazy.cpp | 2 +- src/Databases/DatabaseLazy.h | 2 +- src/Databases/DatabaseOrdinary.cpp | 44 +++++++++++++------ src/Databases/DatabaseOrdinary.h | 3 +- src/Databases/DatabaseReplicated.cpp | 2 +- src/Databases/DatabaseS3.cpp | 2 +- src/Databases/DatabaseS3.h | 2 +- src/Databases/DatabasesCommon.cpp | 4 +- src/Databases/DatabasesCommon.h | 2 +- src/Databases/DatabasesOverlay.cpp | 2 +- src/Databases/DatabasesOverlay.h | 2 +- src/Databases/IDatabase.h | 13 +++++- .../MySQL/DatabaseMaterializedMySQL.cpp | 5 +-- .../MySQL/DatabaseMaterializedMySQL.h | 2 +- src/Databases/MySQL/DatabaseMySQL.cpp | 2 +- src/Databases/MySQL/DatabaseMySQL.h | 2 +- .../DatabaseMaterializedPostgreSQL.cpp | 6 +-- .../DatabaseMaterializedPostgreSQL.h | 2 +- src/Databases/PostgreSQL/DatabasePostgreSQL.h | 2 +- src/Databases/SQLite/DatabaseSQLite.cpp | 2 +- src/Databases/SQLite/DatabaseSQLite.h | 2 +- src/Interpreters/DatabaseCatalog.cpp | 3 ++ src/Interpreters/DatabaseCatalog.h | 6 +-- src/Interpreters/InterpreterDropQuery.cpp | 1 + .../ServerAsynchronousMetrics.cpp | 5 ++- src/Server/ReplicasStatusHandler.cpp | 5 ++- 33 files changed, 85 insertions(+), 56 deletions(-) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 0bf7c8af4b4..37b6123acef 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -416,9 +416,9 @@ void DatabaseAtomic::assertCanBeDetached(bool cleanup) } DatabaseTablesIteratorPtr -DatabaseAtomic::getTablesIterator(ContextPtr local_context, const IDatabase::FilterByNameFunction & filter_by_table_name) const +DatabaseAtomic::getTablesIterator(ContextPtr local_context, const IDatabase::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const { - auto base_iter = DatabaseOrdinary::getTablesIterator(local_context, filter_by_table_name); + auto base_iter = DatabaseOrdinary::getTablesIterator(local_context, filter_by_table_name, skip_not_loaded); return std::make_unique(std::move(typeid_cast(*base_iter))); } diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h index 404478f7cd1..b59edd479ba 100644 --- a/src/Databases/DatabaseAtomic.h +++ b/src/Databases/DatabaseAtomic.h @@ -46,7 +46,7 @@ public: void drop(ContextPtr /*context*/) override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override; diff --git a/src/Databases/DatabaseDictionary.cpp b/src/Databases/DatabaseDictionary.cpp index 76fdb4fa961..adb9a659fcd 100644 --- a/src/Databases/DatabaseDictionary.cpp +++ b/src/Databases/DatabaseDictionary.cpp @@ -80,7 +80,7 @@ StoragePtr DatabaseDictionary::tryGetTable(const String & table_name, ContextPtr return createStorageDictionary(getDatabaseName(), load_result, getContext()); } -DatabaseTablesIteratorPtr DatabaseDictionary::getTablesIterator(ContextPtr, const FilterByNameFunction & filter_by_table_name) const +DatabaseTablesIteratorPtr DatabaseDictionary::getTablesIterator(ContextPtr, const FilterByNameFunction & filter_by_table_name, bool /* skip_not_loaded */) const { return std::make_unique(listTables(filter_by_table_name), getDatabaseName()); } diff --git a/src/Databases/DatabaseDictionary.h b/src/Databases/DatabaseDictionary.h index 469801d183e..a18ea833710 100644 --- a/src/Databases/DatabaseDictionary.h +++ b/src/Databases/DatabaseDictionary.h @@ -34,7 +34,7 @@ public: StoragePtr tryGetTable(const String & table_name, ContextPtr context) const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; bool empty() const override; diff --git a/src/Databases/DatabaseFilesystem.cpp b/src/Databases/DatabaseFilesystem.cpp index 05af0acf978..b27a816a60d 100644 --- a/src/Databases/DatabaseFilesystem.cpp +++ b/src/Databases/DatabaseFilesystem.cpp @@ -229,7 +229,7 @@ std::vector> DatabaseFilesystem::getTablesForBacku * Returns an empty iterator because the database does not have its own tables * But only caches them for quick access */ -DatabaseTablesIteratorPtr DatabaseFilesystem::getTablesIterator(ContextPtr, const FilterByNameFunction &) const +DatabaseTablesIteratorPtr DatabaseFilesystem::getTablesIterator(ContextPtr, const FilterByNameFunction &, bool) const { return std::make_unique(Tables{}, getDatabaseName()); } diff --git a/src/Databases/DatabaseFilesystem.h b/src/Databases/DatabaseFilesystem.h index 3338aa28c21..4b9db5e574d 100644 --- a/src/Databases/DatabaseFilesystem.h +++ b/src/Databases/DatabaseFilesystem.h @@ -45,7 +45,7 @@ public: std::vector> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &, bool) const override; protected: StoragePtr getTableImpl(const String & name, ContextPtr context, bool throw_on_error) const; diff --git a/src/Databases/DatabaseHDFS.h b/src/Databases/DatabaseHDFS.h index b586a912e16..d19918000cf 100644 --- a/src/Databases/DatabaseHDFS.h +++ b/src/Databases/DatabaseHDFS.h @@ -45,7 +45,7 @@ public: void shutdown() override; std::vector> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &, bool) const override; protected: StoragePtr getTableImpl(const String & name, ContextPtr context) const; diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 623c7fbee98..fb1b3ee626b 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -152,7 +152,7 @@ StoragePtr DatabaseLazy::tryGetTable(const String & table_name) const return loadTable(table_name); } -DatabaseTablesIteratorPtr DatabaseLazy::getTablesIterator(ContextPtr, const FilterByNameFunction & filter_by_table_name) const +DatabaseTablesIteratorPtr DatabaseLazy::getTablesIterator(ContextPtr, const FilterByNameFunction & filter_by_table_name, bool /* skip_not_loaded */) const { std::lock_guard lock(mutex); Strings filtered_tables; diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 2b1b119754d..4347649117d 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -62,7 +62,7 @@ public: bool empty() const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; void attachTable(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) override; diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 95bdcfc7dce..3859c2fe0ce 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -438,24 +438,40 @@ void DatabaseOrdinary::stopLoading() stop_load_table.clear(); } -DatabaseTablesIteratorPtr DatabaseOrdinary::getTablesIterator(ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const +DatabaseTablesIteratorPtr DatabaseOrdinary::getTablesIterator(ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const { - // Wait for every table (matching the filter) to be loaded and started up before we make the snapshot. - // It is important, because otherwise table might be: - // - not attached and thus will be missed in the snapshot; - // - not started, which is not good for DDL operations. - LoadTaskPtrs tasks_to_wait; + if (!skip_not_loaded) + { + // Wait for every table (matching the filter) to be loaded and started up before we make the snapshot. + // It is important, because otherwise table might be: + // - not attached and thus will be missed in the snapshot; + // - not started, which is not good for DDL operations. + LoadTaskPtrs tasks_to_wait; + { + std::lock_guard lock(mutex); + if (!filter_by_table_name) + tasks_to_wait.reserve(startup_table.size()); + for (const auto & [table_name, task] : startup_table) + if (!filter_by_table_name || filter_by_table_name(table_name)) + tasks_to_wait.emplace_back(task); + } + waitLoad(currentPoolOr(TablesLoaderForegroundPoolId), tasks_to_wait); + } + return DatabaseWithOwnTablesBase::getTablesIterator(local_context, filter_by_table_name, skip_not_loaded); +} + +Strings DatabaseOrdinary::getAllTableNames(ContextPtr context) const +{ + std::set unique_names; { std::lock_guard lock(mutex); - if (!filter_by_table_name) - tasks_to_wait.reserve(startup_table.size()); - for (const auto & [table_name, task] : startup_table) - if (!filter_by_table_name || filter_by_table_name(table_name)) - tasks_to_wait.emplace_back(task); + for (const auto & [table_name, _] : tables) + unique_names.emplace(table_name); + // Not yet loaded table are not listed in `tables`, so we have to add table names from tasks + for (const auto & [table_name, _] : startup_table) + unique_names.emplace(table_name); } - waitLoad(currentPoolOr(TablesLoaderForegroundPoolId), tasks_to_wait); - - return DatabaseWithOwnTablesBase::getTablesIterator(local_context, filter_by_table_name); + return {unique_names.begin(), unique_names.end()}; } void DatabaseOrdinary::alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 7089540337a..fa5827903cb 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -56,7 +56,8 @@ public: LoadTaskPtr startupDatabaseAsync(AsyncLoader & async_loader, LoadJobSet startup_after, LoadingStrictnessLevel mode) override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; + Strings getAllTableNames(ContextPtr context) const override; void alterTable( ContextPtr context, diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 3b6a712510d..0c2cf1bb011 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1323,13 +1323,13 @@ void DatabaseReplicated::drop(ContextPtr context_) void DatabaseReplicated::stopReplication() { - stopLoading(); if (ddl_worker) ddl_worker->shutdown(); } void DatabaseReplicated::shutdown() { + stopLoading(); stopReplication(); ddl_worker_initialized = false; ddl_worker = nullptr; diff --git a/src/Databases/DatabaseS3.cpp b/src/Databases/DatabaseS3.cpp index 159a5242dbe..1589cc1c75d 100644 --- a/src/Databases/DatabaseS3.cpp +++ b/src/Databases/DatabaseS3.cpp @@ -303,7 +303,7 @@ std::vector> DatabaseS3::getTablesForBackup(const * Returns an empty iterator because the database does not have its own tables * But only caches them for quick access */ -DatabaseTablesIteratorPtr DatabaseS3::getTablesIterator(ContextPtr, const FilterByNameFunction &) const +DatabaseTablesIteratorPtr DatabaseS3::getTablesIterator(ContextPtr, const FilterByNameFunction &, bool) const { return std::make_unique(Tables{}, getDatabaseName()); } diff --git a/src/Databases/DatabaseS3.h b/src/Databases/DatabaseS3.h index 5e7375dbd58..7e38da0fe63 100644 --- a/src/Databases/DatabaseS3.h +++ b/src/Databases/DatabaseS3.h @@ -56,7 +56,7 @@ public: void shutdown() override; std::vector> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &, bool) const override; static Configuration parseArguments(ASTs engine_args, ContextPtr context); diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index f8d6ad69ba8..733af31bdff 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -214,7 +214,7 @@ StoragePtr DatabaseWithOwnTablesBase::tryGetTable(const String & table_name, Con return tryGetTableNoWait(table_name); } -DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(ContextPtr, const FilterByNameFunction & filter_by_table_name) const +DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(ContextPtr, const FilterByNameFunction & filter_by_table_name, bool /* skip_not_loaded */) const { std::lock_guard lock(mutex); if (!filter_by_table_name) @@ -347,7 +347,7 @@ StoragePtr DatabaseWithOwnTablesBase::getTableUnlocked(const String & table_name backQuote(database_name), backQuote(table_name)); } -std::vector> DatabaseWithOwnTablesBase::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const +std::vector> DatabaseWithOwnTablesBase::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool skip_not_loaded) const { std::vector> res; diff --git a/src/Databases/DatabasesCommon.h b/src/Databases/DatabasesCommon.h index 81a3c55a435..2eecf8a564f 100644 --- a/src/Databases/DatabasesCommon.h +++ b/src/Databases/DatabasesCommon.h @@ -35,7 +35,7 @@ public: StoragePtr detachTable(ContextPtr context, const String & table_name) override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; diff --git a/src/Databases/DatabasesOverlay.cpp b/src/Databases/DatabasesOverlay.cpp index c8705254e73..2772db5e066 100644 --- a/src/Databases/DatabasesOverlay.cpp +++ b/src/Databases/DatabasesOverlay.cpp @@ -254,7 +254,7 @@ void DatabasesOverlay::shutdown() db->shutdown(); } -DatabaseTablesIteratorPtr DatabasesOverlay::getTablesIterator(ContextPtr context_, const FilterByNameFunction & filter_by_table_name) const +DatabaseTablesIteratorPtr DatabasesOverlay::getTablesIterator(ContextPtr context_, const FilterByNameFunction & filter_by_table_name, bool /*skip_not_loaded*/) const { Tables tables; for (const auto & db : databases) diff --git a/src/Databases/DatabasesOverlay.h b/src/Databases/DatabasesOverlay.h index b58df506f70..e0c31e009cc 100644 --- a/src/Databases/DatabasesOverlay.h +++ b/src/Databases/DatabasesOverlay.h @@ -52,7 +52,7 @@ public: void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; bool empty() const override; diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 75662bfebe3..b00f2fe4baf 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -229,7 +229,18 @@ public: /// Get an iterator that allows you to pass through all the tables. /// It is possible to have "hidden" tables that are not visible when passing through, but are visible if you get them by name using the functions above. - virtual DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}) const = 0; /// NOLINT + /// Wait for all tables to be loaded and started up. If `skip_not_loaded` is true, then not yet loaded or not yet started up (at the moment of iterator creation) tables are excluded. + virtual DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}, bool skip_not_loaded = false) const = 0; /// NOLINT + + /// Returns list of table names. + virtual Strings getAllTableNames(ContextPtr context) const + { + // NOTE: This default implementation wait for all tables to be loaded and started up. It should be reimplemented for databases that support async loading. + Strings result; + for (auto table_it = getTablesIterator(context); table_it->isValid(); table_it->next()) + result.emplace_back(table_it->name()); + return result; + } /// Is the database empty. virtual bool empty() const = 0; diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp index 0f0d73ae16f..d8360a24bcb 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp @@ -185,9 +185,9 @@ StoragePtr DatabaseMaterializedMySQL::tryGetTable(const String & name, ContextPt } DatabaseTablesIteratorPtr -DatabaseMaterializedMySQL::getTablesIterator(ContextPtr context_, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const +DatabaseMaterializedMySQL::getTablesIterator(ContextPtr context_, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const { - DatabaseTablesIteratorPtr iterator = DatabaseAtomic::getTablesIterator(context_, filter_by_table_name); + DatabaseTablesIteratorPtr iterator = DatabaseAtomic::getTablesIterator(context_, filter_by_table_name, skip_not_loaded); if (context_->isInternalQuery()) return iterator; return std::make_unique(std::move(iterator), this); @@ -201,7 +201,6 @@ void DatabaseMaterializedMySQL::checkIsInternalQuery(ContextPtr context_, const void DatabaseMaterializedMySQL::stopReplication() { - stopLoading(); materialize_thread.stopSynchronization(); started_up = false; } diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.h b/src/Databases/MySQL/DatabaseMaterializedMySQL.h index d2976adcadb..a6418e6fc5c 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.h +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.h @@ -73,7 +73,7 @@ public: StoragePtr tryGetTable(const String & name, ContextPtr context_) const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context_, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context_, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; void checkIsInternalQuery(ContextPtr context_, const char * method) const; diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index d9b0f7f9ac7..b2e199735db 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -105,7 +105,7 @@ bool DatabaseMySQL::empty() const return true; } -DatabaseTablesIteratorPtr DatabaseMySQL::getTablesIterator(ContextPtr local_context, const FilterByNameFunction & filter_by_table_name) const +DatabaseTablesIteratorPtr DatabaseMySQL::getTablesIterator(ContextPtr local_context, const FilterByNameFunction & filter_by_table_name, bool /* skip_not_loaded */) const { Tables tables; std::lock_guard lock(mutex); diff --git a/src/Databases/MySQL/DatabaseMySQL.h b/src/Databases/MySQL/DatabaseMySQL.h index e5b1f434d2f..084a8339be3 100644 --- a/src/Databases/MySQL/DatabaseMySQL.h +++ b/src/Databases/MySQL/DatabaseMySQL.h @@ -58,7 +58,7 @@ public: bool empty() const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_nam, bool skip_not_loaded) const override; ASTPtr getCreateDatabaseQuery() const override; diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index b44bc136b1f..5ef44d3826c 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -455,8 +455,6 @@ void DatabaseMaterializedPostgreSQL::shutdown() void DatabaseMaterializedPostgreSQL::stopReplication() { - stopLoading(); - std::lock_guard lock(handler_mutex); if (replication_handler) replication_handler->shutdown(); @@ -484,10 +482,10 @@ void DatabaseMaterializedPostgreSQL::drop(ContextPtr local_context) DatabaseTablesIteratorPtr DatabaseMaterializedPostgreSQL::getTablesIterator( - ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const + ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const { /// Modify context into nested_context and pass query to Atomic database. - return DatabaseAtomic::getTablesIterator(StorageMaterializedPostgreSQL::makeNestedTableContext(local_context), filter_by_table_name); + return DatabaseAtomic::getTablesIterator(StorageMaterializedPostgreSQL::makeNestedTableContext(local_context), filter_by_table_name, skip_not_loaded); } void registerDatabaseMaterializedPostgreSQL(DatabaseFactory & factory) diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h index dfa53fa61d7..cf1333d03c8 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h @@ -45,7 +45,7 @@ public: void stopLoading() override; DatabaseTablesIteratorPtr - getTablesIterator(ContextPtr context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const override; + getTablesIterator(ContextPtr context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; StoragePtr tryGetTable(const String & name, ContextPtr context) const override; diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index 3ba7333c98e..137b9d5cef9 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -46,7 +46,7 @@ public: void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/) override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; bool isTableExist(const String & name, ContextPtr context) const override; StoragePtr tryGetTable(const String & name, ContextPtr context) const override; diff --git a/src/Databases/SQLite/DatabaseSQLite.cpp b/src/Databases/SQLite/DatabaseSQLite.cpp index b7a82fd9d0f..e758ea35de5 100644 --- a/src/Databases/SQLite/DatabaseSQLite.cpp +++ b/src/Databases/SQLite/DatabaseSQLite.cpp @@ -46,7 +46,7 @@ bool DatabaseSQLite::empty() const } -DatabaseTablesIteratorPtr DatabaseSQLite::getTablesIterator(ContextPtr local_context, const IDatabase::FilterByNameFunction &) const +DatabaseTablesIteratorPtr DatabaseSQLite::getTablesIterator(ContextPtr local_context, const IDatabase::FilterByNameFunction &, bool) const { std::lock_guard lock(mutex); diff --git a/src/Databases/SQLite/DatabaseSQLite.h b/src/Databases/SQLite/DatabaseSQLite.h index e5e93bbc8ce..6bd84a4d297 100644 --- a/src/Databases/SQLite/DatabaseSQLite.h +++ b/src/Databases/SQLite/DatabaseSQLite.h @@ -32,7 +32,7 @@ public: StoragePtr tryGetTable(const String & name, ContextPtr context) const override; - DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override; bool empty() const override; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 9d9f418934f..7231181941a 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -1602,6 +1602,9 @@ void DatabaseCatalog::reloadDisksTask() for (auto & database : getDatabases()) { + // WARNING: In case of `async_load_databases = true` getTablesIterator() call wait for all table in the database to be loaded. + // WARNING: It means that no database will be able to update configuration until all databases are fully loaded. + // TODO: We can split this task by table or by database to make loaded table operate as usual. auto it = database.second->getTablesIterator(getContext()); while (it->isValid()) { diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 6f05a3cea0f..61ec2d9e320 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -418,11 +418,7 @@ public: Names getAllRegisteredNames() const override { - Names result; - if (database) - for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next()) - result.emplace_back(table_it->name()); - return result; + return database ? database->getAllTableNames(context) : {}; } private: diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 72aa4cc63e3..237f49582a6 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -400,6 +400,7 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, table_context->setInternalQuery(true); /// Do not hold extra shared pointers to tables std::vector> tables_to_drop; + // NOTE: This means we wait for all tables to be loaded inside getTablesIterator() call in case of `async_load_databases = true`. for (auto iterator = database->getTablesIterator(table_context); iterator->isValid(); iterator->next()) { auto table_ptr = iterator->table(); diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index fe7ccd64ffe..d26cfcf9fe9 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -278,7 +278,8 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr bool is_system = db.first == DatabaseCatalog::SYSTEM_DATABASE; - for (auto iterator = db.second->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) + // Note that we skip not yet loaded tables, so metrics could possibly be lower than expected on fully loaded database just after server start if `async_load_databases = true`. + for (auto iterator = db.second->getTablesIterator(getContext(), {}, true); iterator->isValid(); iterator->next()) { ++total_number_of_tables; if (is_system) @@ -408,7 +409,7 @@ void ServerAsynchronousMetrics::updateDetachedPartsStats() if (!db.second->canContainMergeTreeTables()) continue; - for (auto iterator = db.second->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) + for (auto iterator = db.second->getTablesIterator(getContext(), {}, true); iterator->isValid(); iterator->next()) { const auto & table = iterator->table(); if (!table) diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 91c6bd722d3..67823117758 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -51,7 +51,10 @@ void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServe if (!db.second->canContainMergeTreeTables()) continue; - for (auto iterator = db.second->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) + // Note that in case `async_load_databases = true` we do not want replica status handler to be hanging + // and waiting (in getTablesIterator() call) for every table to be load, so we just skip not-yet-loaded tables. + // If they have some lag it will be reflected as soon as they are load. + for (auto iterator = db.second->getTablesIterator(getContext(), {}, true); iterator->isValid(); iterator->next()) { const auto & table = iterator->table(); if (!table) From efe5373f2bae7c0101862c7b4e59874d6a332271 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 18 Mar 2024 15:54:18 +0000 Subject: [PATCH 0186/1165] refactoring of reading from compact parts --- .../MergeTree/MergeTreeDataPartCompact.cpp | 23 +- src/Storages/MergeTree/MergeTreeIOSettings.h | 13 + .../MergeTree/MergeTreeIndexReader.cpp | 21 +- .../MergeTree/MergeTreeMarksLoader.cpp | 78 +-- src/Storages/MergeTree/MergeTreeMarksLoader.h | 51 +- .../MergeTree/MergeTreeReaderCompact.cpp | 515 ++++-------------- .../MergeTree/MergeTreeReaderCompact.h | 77 +-- .../MergeTreeReaderCompactSingleBuffer.cpp | 108 ++++ .../MergeTreeReaderCompactSingleBuffer.h | 33 ++ .../MergeTree/MergeTreeReaderStream.cpp | 441 ++++++++++----- .../MergeTree/MergeTreeReaderStream.h | 136 ++++- .../MergeTree/MergeTreeReaderWide.cpp | 27 +- src/Storages/StorageMergeTreeIndex.cpp | 4 +- 13 files changed, 814 insertions(+), 713 deletions(-) create mode 100644 src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp create mode 100644 src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.h diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 9f201ab3b81..77a73fcc8f9 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -1,10 +1,8 @@ #include "MergeTreeDataPartCompact.h" #include -#include +#include #include -#include #include -#include namespace DB @@ -41,21 +39,12 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( const ReadBufferFromFileBase::ProfileCallback & profile_callback) const { auto read_info = std::make_shared(shared_from_this(), alter_conversions); - auto * load_marks_threadpool - = reader_settings.read_settings.load_marks_asynchronously ? &read_info->getContext()->getLoadMarksThreadpool() : nullptr; - return std::make_unique( - read_info, - columns_to_read, - virtual_fields, - storage_snapshot, - uncompressed_cache, - mark_cache, - mark_ranges, - reader_settings, - load_marks_threadpool, - avg_value_size_hints, - profile_callback); + return std::make_unique( + read_info, columns_to_read, virtual_fields, + storage_snapshot, uncompressed_cache, + mark_cache, mark_ranges, reader_settings, + avg_value_size_hints, profile_callback, CLOCK_MONOTONIC_COARSE); } IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 379d7af8569..feeb1808a6f 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -13,6 +13,11 @@ namespace DB class MMappedFileCache; using MMappedFileCachePtr = std::shared_ptr; +enum class CompactPartsReadMethod +{ + SingleBuffer, + MultiBuffer, +}; struct MergeTreeReaderSettings { @@ -25,12 +30,20 @@ struct MergeTreeReaderSettings bool checksum_on_read = true; /// True if we read in order of sorting key. bool read_in_order = false; + /// Use one buffer for each column or for all columns while reading from compact. + CompactPartsReadMethod compact_parts_read_method = CompactPartsReadMethod::SingleBuffer; + /// True if we read stream for dictionary of LowCardinality type. + bool is_low_cardinality_dictionary = false; + /// True if data may be compressed by different codecs in one stream. + bool allow_different_codecs = false; /// Deleted mask is applied to all reads except internal select from mutate some part columns. bool apply_deleted_mask = true; /// Put reading task in a common I/O pool, return Async state on prepare() bool use_asynchronous_read_from_pool = false; /// If PREWHERE has multiple conditions combined with AND, execute them in separate read/filtering steps. bool enable_multiple_prewhere_read_steps = false; + /// If true, try to lower size of read buffer according to granule size and compressed block size. + bool adjust_read_buffer_size = true; }; struct MergeTreeWriterSettings diff --git a/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/src/Storages/MergeTree/MergeTreeIndexReader.cpp index f0b1aacc239..6012994b46d 100644 --- a/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -20,14 +20,23 @@ std::unique_ptr makeIndexReader( auto context = part->storage.getContext(); auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; - return std::make_unique( + auto marks_loader = std::make_shared( std::make_shared(part, std::make_shared()), + mark_cache, + part->index_granularity_info.getMarksFilePath(index->getFileName()), + marks_count, + part->index_granularity_info, + settings.save_marks_in_cache, + settings.read_settings, + load_marks_threadpool, + /*num_columns_in_mark=*/ 1); + + return std::make_unique( + part->getDataPartStoragePtr(), index->getFileName(), extension, marks_count, - all_mark_ranges, - std::move(settings), mark_cache, uncompressed_cache, - part->getFileSizeOrZero(index->getFileName() + extension), - &part->index_granularity_info, - ReadBufferFromFileBase::ProfileCallback{}, CLOCK_MONOTONIC_COARSE, false, load_marks_threadpool); + all_mark_ranges, std::move(settings), uncompressed_cache, + part->getFileSizeOrZero(index->getFileName() + extension), std::move(marks_loader), + ReadBufferFromFileBase::ProfileCallback{}, CLOCK_MONOTONIC_COARSE); } } diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 8250050412f..a7a635f3b72 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -28,6 +28,23 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +MergeTreeMarksGetter::MergeTreeMarksGetter(MarkCache::MappedPtr marks_, size_t num_columns_in_mark_) + : marks(std::move(marks_)), num_columns_in_mark(num_columns_in_mark_) +{ + assert(marks); +} + +MarkInCompressedFile MergeTreeMarksGetter::getMark(size_t row_index, size_t column_index) const +{ +#ifndef NDEBUG + if (column_index >= num_columns_in_mark) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Column index: {} is out of range [0, {})", column_index, num_columns_in_mark); +#endif + + return marks->get(row_index * num_columns_in_mark + column_index); +} + MergeTreeMarksLoader::MergeTreeMarksLoader( MergeTreeDataPartInfoForReaderPtr data_part_reader_, MarkCache * mark_cache_, @@ -37,58 +54,49 @@ MergeTreeMarksLoader::MergeTreeMarksLoader( bool save_marks_in_cache_, const ReadSettings & read_settings_, ThreadPool * load_marks_threadpool_, - size_t columns_in_mark_) + size_t num_columns_in_mark_) : data_part_reader(data_part_reader_) , mark_cache(mark_cache_) , mrk_path(mrk_path_) , marks_count(marks_count_) , index_granularity_info(index_granularity_info_) , save_marks_in_cache(save_marks_in_cache_) - , columns_in_mark(columns_in_mark_) , read_settings(read_settings_) + , num_columns_in_mark(num_columns_in_mark_) , load_marks_threadpool(load_marks_threadpool_) { if (load_marks_threadpool) - { future = loadMarksAsync(); - } } MergeTreeMarksLoader::~MergeTreeMarksLoader() { if (future.valid()) - { future.wait(); - } } - -MarkInCompressedFile MergeTreeMarksLoader::getMark(size_t row_index, size_t column_index) +MergeTreeMarksGetterPtr MergeTreeMarksLoader::loadMarks() { - if (!marks) + std::lock_guard lock(load_mutex); + + if (marks) + return std::make_unique(marks, num_columns_in_mark); + + Stopwatch watch(CLOCK_MONOTONIC); + + if (future.valid()) { - Stopwatch watch(CLOCK_MONOTONIC); - - if (future.valid()) - { - marks = future.get(); - future = {}; - } - else - { - marks = loadMarks(); - } - - watch.stop(); - ProfileEvents::increment(ProfileEvents::WaitMarksLoadMicroseconds, watch.elapsedMicroseconds()); + marks = future.get(); + future = {}; + } + else + { + marks = loadMarksSync(); } -#ifndef NDEBUG - if (column_index >= columns_in_mark) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Column index: {} is out of range [0, {})", column_index, columns_in_mark); -#endif - - return marks->get(row_index * columns_in_mark + column_index); + watch.stop(); + ProfileEvents::increment(ProfileEvents::WaitMarksLoadMicroseconds, watch.elapsedMicroseconds()); + return std::make_unique(marks, num_columns_in_mark); } @@ -100,12 +108,12 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() auto data_part_storage = data_part_reader->getDataPartStorage(); size_t file_size = data_part_storage->getFileSize(mrk_path); - size_t mark_size = index_granularity_info.getMarkSizeInBytes(columns_in_mark); + size_t mark_size = index_granularity_info.getMarkSizeInBytes(num_columns_in_mark); size_t expected_uncompressed_size = mark_size * marks_count; // We first read the marks into a temporary simple array, then compress them into a more compact // representation. - PODArray plain_marks(marks_count * columns_in_mark); // temporary + PODArray plain_marks(marks_count * num_columns_in_mark); // temporary auto full_mark_path = std::string(fs::path(data_part_storage->getFullPath()) / mrk_path); if (file_size == 0 && marks_count != 0) @@ -159,7 +167,7 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() size_t granularity; reader->readStrict( - reinterpret_cast(plain_marks.data() + i * columns_in_mark), columns_in_mark * sizeof(MarkInCompressedFile)); + reinterpret_cast(plain_marks.data() + i * num_columns_in_mark), num_columns_in_mark * sizeof(MarkInCompressedFile)); readBinaryLittleEndian(granularity, *reader); } @@ -182,13 +190,13 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() auto res = std::make_shared(plain_marks); - ProfileEvents::increment(ProfileEvents::LoadedMarksCount, marks_count * columns_in_mark); + ProfileEvents::increment(ProfileEvents::LoadedMarksCount, marks_count * num_columns_in_mark); ProfileEvents::increment(ProfileEvents::LoadedMarksMemoryBytes, res->approximateMemoryUsage()); return res; } -MarkCache::MappedPtr MergeTreeMarksLoader::loadMarks() +MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksSync() { MarkCache::MappedPtr loaded_marks; @@ -227,7 +235,7 @@ std::future MergeTreeMarksLoader::loadMarksAsync() [this]() -> MarkCache::MappedPtr { ProfileEvents::increment(ProfileEvents::BackgroundLoadingMarksTasks); - return loadMarks(); + return loadMarksSync(); }, *load_marks_threadpool, "LoadMarksThread"); diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index 0889da0cb85..73dd462f2fa 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -10,13 +10,33 @@ namespace DB { struct MergeTreeIndexGranularityInfo; +using MarksPtr = MarkCache::MappedPtr; class Threadpool; +/// Class that helps to get marks by indexes. +/// Always immutable and thread safe. +/// Marks can be shared between several threads +/// that read columns from the same file. +class MergeTreeMarksGetter +{ +public: + MergeTreeMarksGetter(MarkCache::MappedPtr marks_, size_t num_columns_in_mark_); + + MarkInCompressedFile getMark(size_t row_index, size_t column_index) const; + size_t getNumColumns() const { return num_columns_in_mark; } + +private: + const MarkCache::MappedPtr marks; + const size_t num_columns_in_mark; +}; + +using MergeTreeMarksGetterPtr = std::unique_ptr; + +/// Class that helps to load marks on demand. +/// Thread safe, but locks while loading marks. class MergeTreeMarksLoader { public: - using MarksPtr = MarkCache::MappedPtr; - MergeTreeMarksLoader( MergeTreeDataPartInfoForReaderPtr data_part_reader_, MarkCache * mark_cache_, @@ -26,24 +46,27 @@ public: bool save_marks_in_cache_, const ReadSettings & read_settings_, ThreadPool * load_marks_threadpool_, - size_t columns_in_mark_ = 1); + size_t num_columns_in_mark_); ~MergeTreeMarksLoader(); - MarkInCompressedFile getMark(size_t row_index, size_t column_index = 0); + MergeTreeMarksGetterPtr loadMarks(); + size_t getNumColumns() const { return num_columns_in_mark; } private: - MergeTreeDataPartInfoForReaderPtr data_part_reader; - MarkCache * mark_cache = nullptr; - String mrk_path; - size_t marks_count; + const MergeTreeDataPartInfoForReaderPtr data_part_reader; + MarkCache * const mark_cache; + const String mrk_path; + const size_t marks_count; const MergeTreeIndexGranularityInfo & index_granularity_info; - bool save_marks_in_cache = false; - size_t columns_in_mark; - MarkCache::MappedPtr marks; - ReadSettings read_settings; + const bool save_marks_in_cache; + const ReadSettings read_settings; + const size_t num_columns_in_mark; - MarkCache::MappedPtr loadMarks(); + std::mutex load_mutex; + MarkCache::MappedPtr marks; + + MarkCache::MappedPtr loadMarksSync(); std::future loadMarksAsync(); MarkCache::MappedPtr loadMarksImpl(); @@ -51,4 +74,6 @@ private: ThreadPool * load_marks_threadpool; }; +using MergeTreeMarksLoaderPtr = std::shared_ptr; + } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 63824366722..3835c1e2054 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -10,10 +10,8 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_READ_ALL_DATA; - extern const int ARGUMENT_OUT_OF_BOUND; } - MergeTreeReaderCompact::MergeTreeReaderCompact( MergeTreeDataPartInfoForReaderPtr data_part_info_for_read_, NamesAndTypesList columns_, @@ -23,7 +21,6 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( MarkCache * mark_cache_, MarkRanges mark_ranges_, MergeTreeReaderSettings settings_, - ThreadPool * load_marks_threadpool_, ValueSizeMap avg_value_size_hints_, const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) @@ -37,91 +34,22 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( mark_ranges_, settings_, avg_value_size_hints_) - , marks_loader( - data_part_info_for_read_, - mark_cache, - data_part_info_for_read_->getIndexGranularityInfo().getMarksFilePath(MergeTreeDataPartCompact::DATA_FILE_NAME), - data_part_info_for_read_->getMarksCount(), - data_part_info_for_read_->getIndexGranularityInfo(), - settings.save_marks_in_cache, - settings.read_settings, - load_marks_threadpool_, - data_part_info_for_read_->getColumns().size()) + , marks_loader(std::make_shared( + data_part_info_for_read_, + mark_cache, + data_part_info_for_read_->getIndexGranularityInfo().getMarksFilePath(MergeTreeDataPartCompact::DATA_FILE_NAME), + data_part_info_for_read_->getMarksCount(), + data_part_info_for_read_->getIndexGranularityInfo(), + settings.save_marks_in_cache, + settings.read_settings, + settings_.read_settings.load_marks_asynchronously + ? &data_part_info_for_read_->getContext()->getLoadMarksThreadpool() : nullptr, + data_part_info_for_read_->getColumns().size())) , profile_callback(profile_callback_) , clock_type(clock_type_) { } -void MergeTreeReaderCompact::initialize() -{ - try - { - fillColumnPositions(); - - /// Do not use max_read_buffer_size, but try to lower buffer size with maximal size of granule to avoid reading much data. - auto buffer_size = getReadBufferSize(*data_part_info_for_read, marks_loader, column_positions, all_mark_ranges); - if (buffer_size) - settings.read_settings = settings.read_settings.adjustBufferSize(buffer_size); - - if (!settings.read_settings.local_fs_buffer_size || !settings.read_settings.remote_fs_buffer_size) - throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read to empty buffer."); - - const String path = MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION; - auto data_part_storage = data_part_info_for_read->getDataPartStorage(); - - if (uncompressed_cache) - { - auto buffer = std::make_unique( - std::string(fs::path(data_part_storage->getFullPath()) / path), - [this, path, data_part_storage]() - { - return data_part_storage->readFile( - path, - settings.read_settings, - std::nullopt, std::nullopt); - }, - uncompressed_cache, - /* allow_different_codecs = */ true); - - if (profile_callback) - buffer->setProfileCallback(profile_callback, clock_type); - - if (!settings.checksum_on_read) - buffer->disableChecksumming(); - - cached_buffer = std::move(buffer); - data_buffer = cached_buffer.get(); - compressed_data_buffer = cached_buffer.get(); - } - else - { - auto buffer = - std::make_unique( - data_part_storage->readFile( - path, - settings.read_settings, - std::nullopt, std::nullopt), - /* allow_different_codecs = */ true); - - if (profile_callback) - buffer->setProfileCallback(profile_callback, clock_type); - - if (!settings.checksum_on_read) - buffer->disableChecksumming(); - - non_cached_buffer = std::move(buffer); - data_buffer = non_cached_buffer.get(); - compressed_data_buffer = non_cached_buffer.get(); - } - } - catch (...) - { - if (!isRetryableException(std::current_exception())) - data_part_info_for_read->reportBroken(); - throw; - } -} - void MergeTreeReaderCompact::fillColumnPositions() { size_t columns_num = columns_to_read.size(); @@ -191,189 +119,31 @@ void MergeTreeReaderCompact::fillColumnPositions() } } -size_t MergeTreeReaderCompact::readRows( - size_t from_mark, size_t current_task_last_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) -{ - if (!initialized) - { - initialize(); - initialized = true; - } - - if (continue_reading) - from_mark = next_mark; - - size_t read_rows = 0; - size_t num_columns = columns_to_read.size(); - checkNumberOfColumns(num_columns); - - MutableColumns mutable_columns(num_columns); - for (size_t i = 0; i < num_columns; ++i) - { - if (column_positions[i] && res_columns[i] == nullptr) - res_columns[i] = columns_to_read[i].type->createColumn(*serializations[i]); - } - - while (read_rows < max_rows_to_read) - { - size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); - - /// If we need to read multiple subcolumns from a single column in storage, - /// we will read it this column only once and then reuse to extract all subcolumns. - std::unordered_map columns_cache_for_subcolumns; - - for (size_t pos = 0; pos < num_columns; ++pos) - { - if (!res_columns[pos]) - continue; - - try - { - auto & column = res_columns[pos]; - size_t column_size_before_reading = column->size(); - - readData(columns_to_read[pos], column, from_mark, current_task_last_mark, *column_positions[pos], rows_to_read, columns_for_offsets[pos], columns_cache_for_subcolumns); - - size_t read_rows_in_column = column->size() - column_size_before_reading; - if (read_rows_in_column != rows_to_read) - throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, - "Cannot read all data in MergeTreeReaderCompact. Rows read: {}. Rows expected: {}.", - read_rows_in_column, rows_to_read); - } - catch (...) - { - if (!isRetryableException(std::current_exception())) - data_part_info_for_read->reportBroken(); - - /// Better diagnostics. - try - { - rethrow_exception(std::current_exception()); - } - catch (Exception & e) - { - e.addMessage(getMessageForDiagnosticOfBrokenPart(from_mark, max_rows_to_read)); - } - - throw; - } - } - - ++from_mark; - read_rows += rows_to_read; - } - - next_mark = from_mark; - - return read_rows; -} - void MergeTreeReaderCompact::readData( - const NameAndTypePair & name_and_type, ColumnPtr & column, - size_t from_mark, size_t current_task_last_mark, size_t column_position, size_t rows_to_read, - ColumnNameLevel name_level_for_offsets, std::unordered_map & columns_cache_for_subcolumns) + const NameAndTypePair & name_and_type, + ColumnPtr & column, + size_t rows_to_read, + const InputStreamGetter & getter) { - const auto & [name, type] = name_and_type; - std::optional column_for_offsets; - - if (name_level_for_offsets.has_value()) + try { - const auto & part_columns = data_part_info_for_read->getColumnsDescription(); - column_for_offsets = part_columns.getPhysical(name_level_for_offsets->first); - } + const auto [name, type] = name_and_type; + size_t column_size_before_reading = column->size(); - adjustUpperBound(current_task_last_mark); /// Must go before seek. + ISerialization::DeserializeBinaryBulkSettings deserialize_settings; + deserialize_settings.getter = getter; + deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; - if (!isContinuousReading(from_mark, column_position)) - seekToMark(from_mark, column_position); - - /// If we read only offsets we have to read prefix anyway - /// to preserve correctness of serialization. - auto buffer_getter_for_prefix = [&](const auto &) -> ReadBuffer * - { - return data_buffer; - }; - - auto buffer_getter = [&](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer * - { - /// Offset stream from another column could be read, in case of current - /// column does not exists (see findColumnForOffsets() in - /// MergeTreeReaderCompact::fillColumnPositions()) - if (name_level_for_offsets.has_value()) + if (name_and_type.isSubcolumn()) { - bool is_offsets = !substream_path.empty() && substream_path.back().type == ISerialization::Substream::ArraySizes; - if (!is_offsets) - return nullptr; + const auto & type_in_storage = name_and_type.getTypeInStorage(); + const auto & name_in_storage = name_and_type.getNameInStorage(); - /// Offset stream can be read only from columns of current level or - /// below (since it is OK to read all parent streams from the - /// alternative). - /// - /// Consider the following columns in nested "root": - /// - root.array Array(UInt8) - exists - /// - root.nested_array Array(Array(UInt8)) - does not exists (only_offsets_level=1) - /// - /// For root.nested_array it will try to read multiple streams: - /// - offsets (substream_path = {ArraySizes}) - /// OK - /// - root.nested_array elements (substream_path = {ArrayElements, ArraySizes}) - /// NOT OK - cannot use root.array offsets stream for this - /// - /// Here only_offsets_level is the level of the alternative stream, - /// and substream_path.size() is the level of the current stream. - if (name_level_for_offsets->second < ISerialization::getArrayLevel(substream_path)) - return nullptr; - } + auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); + ColumnPtr temp_column = type_in_storage->createColumn(*serialization); - return data_buffer; - }; - - ISerialization::DeserializeBinaryBulkStatePtr state; - ISerialization::DeserializeBinaryBulkStatePtr state_for_prefix; - - ISerialization::DeserializeBinaryBulkSettings deserialize_settings; - deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; - bool columns_cache_was_used = false; - - if (name_and_type.isSubcolumn()) - { - NameAndTypePair name_type_in_storage{name_and_type.getNameInStorage(), name_and_type.getTypeInStorage()}; - ColumnPtr temp_column; - - auto it = columns_cache_for_subcolumns.find(name_type_in_storage.name); - if (!column_for_offsets && it != columns_cache_for_subcolumns.end()) - { - temp_column = it->second; - auto subcolumn = name_type_in_storage.type->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); - if (column->empty()) - column = IColumn::mutate(subcolumn); - else - column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); - - columns_cache_was_used = true; - } - else - { - /// In case of reading only offset use the correct serialization for reading of the prefix - auto serialization = getSerializationInPart(name_type_in_storage); - temp_column = name_type_in_storage.type->createColumn(*serialization); - - if (column_for_offsets) - { - auto serialization_for_prefix = getSerializationInPart(*column_for_offsets); - - deserialize_settings.getter = buffer_getter_for_prefix; - serialization_for_prefix->deserializeBinaryBulkStatePrefix(deserialize_settings, state_for_prefix); - } - - deserialize_settings.getter = buffer_getter; - serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, state); - serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, state, nullptr); - - if (!column_for_offsets) - columns_cache_for_subcolumns[name_type_in_storage.name] = temp_column; - - auto subcolumn = name_type_in_storage.type->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); + serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); + auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); /// TODO: Avoid extra copying. if (column->empty()) @@ -381,185 +151,98 @@ void MergeTreeReaderCompact::readData( else column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); } - } - else - { - /// In case of reading only offsets use the correct serialization for reading the prefix - auto serialization = getSerializationInPart(name_and_type); - - if (column_for_offsets) + else { - auto serialization_for_prefix = getSerializationInPart(*column_for_offsets); - - deserialize_settings.getter = buffer_getter_for_prefix; - serialization_for_prefix->deserializeBinaryBulkStatePrefix(deserialize_settings, state_for_prefix); + auto serialization = getSerializationInPart(name_and_type); + serialization->deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); } - deserialize_settings.getter = buffer_getter; - serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, state); - serialization->deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, state, nullptr); - } - - /// The buffer is left in inconsistent state after reading single offsets or using columns cache during subcolumns reading. - if (name_level_for_offsets.has_value() || columns_cache_was_used) - last_read_granule.reset(); - else - last_read_granule.emplace(from_mark, column_position); -} - -void MergeTreeReaderCompact::prefetchBeginOfRange(Priority priority) -try -{ - if (!initialized) - { - initialize(); - initialized = true; - } - - adjustUpperBound(all_mark_ranges.back().end); - seekToMark(all_mark_ranges.front().begin, 0); - data_buffer->prefetch(priority); -} -catch (...) -{ - if (!isRetryableException(std::current_exception())) - data_part_info_for_read->reportBroken(); - throw; -} - -void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) -{ - MarkInCompressedFile mark = marks_loader.getMark(row_index, column_index); - try - { - compressed_data_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + size_t read_rows_in_column = column->size() - column_size_before_reading; + if (read_rows_in_column != rows_to_read) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read all data in MergeTreeReaderCompact. Rows read: {}. Rows expected: {}.", + read_rows_in_column, rows_to_read); } catch (Exception & e) { - /// Better diagnostics. - if (e.code() == ErrorCodes::ARGUMENT_OUT_OF_BOUND) - e.addMessage("(while seeking to mark (" + toString(row_index) + ", " + toString(column_index) + ")"); - + e.addMessage("(while reading column " + name_and_type.name + ")"); throw; } } -void MergeTreeReaderCompact::adjustUpperBound(size_t last_mark) + +void MergeTreeReaderCompact::readPrefix( + const NameAndTypePair & name_and_type, + const InputStreamGetter & buffer_getter, + const InputStreamGetter & buffer_getter_for_prefix, + const ColumnNameLevel & name_level_for_offsets) { - size_t right_offset = 0; - if (last_mark < data_part_info_for_read->getMarksCount()) /// Otherwise read until the end of file - right_offset = marks_loader.getMark(last_mark).offset_in_compressed_file; - - if (right_offset == 0) + try { - /// If already reading till the end of file. - if (last_right_offset && *last_right_offset == 0) - return; + ISerialization::DeserializeBinaryBulkSettings deserialize_settings; - last_right_offset = 0; // Zero value means the end of file. - data_buffer->setReadUntilEnd(); + if (name_level_for_offsets.has_value()) + { + const auto & part_columns = data_part_info_for_read->getColumnsDescription(); + auto column_for_offsets = part_columns.getPhysical(name_level_for_offsets->first); + + auto serialization_for_prefix = getSerializationInPart(column_for_offsets); + deserialize_settings.getter = buffer_getter_for_prefix; + ISerialization::DeserializeBinaryBulkStatePtr state_for_prefix; + + serialization_for_prefix->deserializeBinaryBulkStatePrefix(deserialize_settings, state_for_prefix); + } + + SerializationPtr serialization; + if (name_and_type.isSubcolumn()) + serialization = getSerializationInPart({name_and_type.getNameInStorage(), name_and_type.getTypeInStorage()}); + else + serialization = getSerializationInPart(name_and_type); + + + deserialize_settings.getter = buffer_getter; + serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.name]); } - else + catch (Exception & e) { - if (last_right_offset && right_offset <= last_right_offset.value()) - return; - - last_right_offset = right_offset; - data_buffer->setReadUntilPosition(right_offset); + e.addMessage("(while reading column " + name_and_type.name + ")"); + throw; } } -bool MergeTreeReaderCompact::isContinuousReading(size_t mark, size_t column_position) +void MergeTreeReaderCompact::createColumnsForReading(Columns & res_columns) const { - if (!last_read_granule) + for (size_t i = 0; i < columns_to_read.size(); ++i) + { + if (column_positions[i] && res_columns[i] == nullptr) + res_columns[i] = columns_to_read[i].type->createColumn(*serializations[i]); + } +} + +bool MergeTreeReaderCompact::needSkipStream(size_t column_pos, const ISerialization::SubstreamPath & substream) const +{ + /// Offset stream can be read only from columns of current level or + /// below (since it is OK to read all parent streams from the + /// alternative). + /// + /// Consider the following columns in nested "root": + /// - root.array Array(UInt8) - exists + /// - root.nested_array Array(Array(UInt8)) - does not exists (only_offsets_level=1) + /// + /// For root.nested_array it will try to read multiple streams: + /// - offsets (substream_path = {ArraySizes}) + /// OK + /// - root.nested_array elements (substream_path = {ArrayElements, ArraySizes}) + /// NOT OK - cannot use root.array offsets stream for this + /// + /// Here only_offsets_level is the level of the alternative stream, + /// and substream_path.size() is the level of the current stream. + + if (!columns_for_offsets[column_pos]) return false; - const auto & [last_mark, last_column] = *last_read_granule; - return (mark == last_mark && column_position == last_column + 1) - || (mark == last_mark + 1 && column_position == 0 && last_column == data_part_info_for_read->getColumns().size() - 1); -} -namespace -{ - -/// A simple class that helps to iterate over 2-dim marks of compact parts. -class MarksCounter -{ -public: - MarksCounter(size_t rows_num_, size_t columns_num_) - : rows_num(rows_num_), columns_num(columns_num_) {} - - struct Iterator - { - size_t row; - size_t column; - MarksCounter * counter; - - Iterator(size_t row_, size_t column_, MarksCounter * counter_) - : row(row_), column(column_), counter(counter_) {} - - Iterator operator++() - { - if (column + 1 == counter->columns_num) - { - ++row; - column = 0; - } - else - { - ++column; - } - - return *this; - } - - bool operator==(const Iterator & other) const { return row == other.row && column == other.column; } - bool operator!=(const Iterator & other) const { return !(*this == other); } - }; - - Iterator get(size_t row, size_t column) { return Iterator(row, column, this); } - Iterator end() { return get(rows_num, 0); } - -private: - size_t rows_num; - size_t columns_num; -}; - -} - -size_t MergeTreeReaderCompact::getReadBufferSize( - const IMergeTreeDataPartInfoForReader & data_part_info_for_reader, - MergeTreeMarksLoader & marks_loader, - const ColumnPositions & column_positions, - const MarkRanges & mark_ranges) -{ - size_t buffer_size = 0; - size_t columns_num = column_positions.size(); - size_t file_size = data_part_info_for_reader.getFileSizeOrZero(MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION); - - MarksCounter counter(data_part_info_for_reader.getMarksCount(), data_part_info_for_reader.getColumns().size()); - - for (const auto & mark_range : mark_ranges) - { - for (size_t mark = mark_range.begin; mark < mark_range.end; ++mark) - { - for (size_t i = 0; i < columns_num; ++i) - { - if (!column_positions[i]) - continue; - - auto it = counter.get(mark, *column_positions[i]); - size_t cur_offset = marks_loader.getMark(it.row, it.column).offset_in_compressed_file; - - while (it != counter.end() && cur_offset == marks_loader.getMark(it.row, it.column).offset_in_compressed_file) - ++it; - - size_t next_offset = (it == counter.end() ? file_size : marks_loader.getMark(it.row, it.column).offset_in_compressed_file); - buffer_size = std::max(buffer_size, next_offset - cur_offset); - } - } - } - - return buffer_size; + bool is_offsets = !substream.empty() && substream.back().type == ISerialization::Substream::ArraySizes; + return !is_offsets || columns_for_offsets[column_pos]->second < ISerialization::getArrayLevel(substream); } } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 769e6a08be4..9c66b4691a3 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -3,7 +3,7 @@ #include #include #include - +#include namespace DB { @@ -14,7 +14,7 @@ using DataPartCompactPtr = std::shared_ptr; class IMergeTreeDataPart; using DataPartPtr = std::shared_ptr; -/// Reader for compact parts +/// Base class of readers for compact parts. class MergeTreeReaderCompact : public IMergeTreeReader { public: @@ -27,31 +27,37 @@ public: MarkCache * mark_cache_, MarkRanges mark_ranges_, MergeTreeReaderSettings settings_, - ThreadPool * load_marks_threadpool_, - ValueSizeMap avg_value_size_hints_ = {}, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = {}, - clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE); + ValueSizeMap avg_value_size_hints_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, + clockid_t clock_type_); - /// Return the number of rows has been read or zero if there is no columns to read. - /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark - size_t readRows(size_t from_mark, size_t current_task_last_mark, - bool continue_reading, size_t max_rows_to_read, Columns & res_columns) override; + bool canReadIncompleteGranules() const override final { return false; } - bool canReadIncompleteGranules() const override { return false; } - - void prefetchBeginOfRange(Priority priority) override; - -private: - bool isContinuousReading(size_t mark, size_t column_position); +protected: void fillColumnPositions(); - void initialize(); - ReadBuffer * data_buffer; - CompressedReadBufferBase * compressed_data_buffer; - std::unique_ptr cached_buffer; - std::unique_ptr non_cached_buffer; + using InputStreamGetter = ISerialization::InputStreamGetter; - MergeTreeMarksLoader marks_loader; + void readData( + const NameAndTypePair & name_and_type, + ColumnPtr & column, + size_t rows_to_read, + const InputStreamGetter & getter); + + void readPrefix( + const NameAndTypePair & name_and_type, + const InputStreamGetter & buffer_getter, + const InputStreamGetter & buffer_getter_for_prefix, + const ColumnNameLevel & name_level_for_offsets); + + void createColumnsForReading(Columns & res_columns) const; + bool needSkipStream(size_t column_pos, const ISerialization::SubstreamPath & substream) const; + + const MergeTreeMarksLoaderPtr marks_loader; + MergeTreeMarksGetterPtr marks_getter; + + ReadBufferFromFileBase::ProfileCallback profile_callback; + clockid_t clock_type; /// Storage columns with collected separate arrays of Nested to columns of Nested type. /// They maybe be needed for finding offsets of missed Nested columns in parts. @@ -67,32 +73,9 @@ private: /// Element of the vector is the level of the alternative stream. std::vector columns_for_offsets; - /// For asynchronous reading from remote fs. Same meaning as in MergeTreeReaderStream. - std::optional last_right_offset; - + /// Mark to read in next 'readRows' call in case, + /// when 'continue_reading' is true. size_t next_mark = 0; - std::optional> last_read_granule; - - void seekToMark(size_t row_index, size_t column_index); - - void readData(const NameAndTypePair & name_and_type, ColumnPtr & column, size_t from_mark, - size_t current_task_last_mark, size_t column_position, - size_t rows_to_read, ColumnNameLevel name_level_for_offsets, std::unordered_map & columns_cache_for_subcolumns); - - /// Returns maximal value of granule size in compressed file from @mark_ranges. - /// This value is used as size of read buffer. - static size_t getReadBufferSize( - const IMergeTreeDataPartInfoForReader & data_part_info_for_reader, - MergeTreeMarksLoader & marks_loader, - const ColumnPositions & column_positions, - const MarkRanges & mark_ranges); - - /// For asynchronous reading from remote fs. - void adjustUpperBound(size_t last_mark); - - ReadBufferFromFileBase::ProfileCallback profile_callback; - clockid_t clock_type; - bool initialized = false; }; } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp new file mode 100644 index 00000000000..2b2cf493bb5 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -0,0 +1,108 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +size_t MergeTreeReaderCompactSingleBuffer::readRows( + size_t from_mark, size_t current_task_last_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) +try +{ + init(); + + if (continue_reading) + from_mark = next_mark; + + size_t read_rows = 0; + size_t num_columns = columns_to_read.size(); + + checkNumberOfColumns(num_columns); + createColumnsForReading(res_columns); + + while (read_rows < max_rows_to_read) + { + size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); + + for (size_t pos = 0; pos < num_columns; ++pos) + { + if (!res_columns[pos]) + continue; + + auto & column = res_columns[pos]; + + stream->adjustRightMark(current_task_last_mark); /// Must go before seek. + stream->seekToMarkAndColumn(from_mark, *column_positions[pos]); + + auto buffer_getter = [&](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer * + { + if (needSkipStream(pos, substream_path)) + return nullptr; + + return stream->getDataBuffer(); + }; + + /// If we read only offsets we have to read prefix anyway + /// to preserve correctness of serialization. + auto buffer_getter_for_prefix = [&](const auto &) -> ReadBuffer * + { + return stream->getDataBuffer(); + }; + + readPrefix(columns_to_read[pos], buffer_getter, buffer_getter_for_prefix, columns_for_offsets[pos]); + readData(columns_to_read[pos], column, rows_to_read, buffer_getter); + } + + ++from_mark; + read_rows += rows_to_read; + } + + next_mark = from_mark; + return read_rows; +} +catch (...) +{ + if (!isRetryableException(std::current_exception())) + data_part_info_for_read->reportBroken(); + + /// Better diagnostics. + try + { + rethrow_exception(std::current_exception()); + } + catch (Exception & e) + { + e.addMessage(getMessageForDiagnosticOfBrokenPart(from_mark, max_rows_to_read)); + } + + throw; +} + +void MergeTreeReaderCompactSingleBuffer::init() +try +{ + if (initialized) + return; + + auto stream_settings = settings; + stream_settings.allow_different_codecs = true; + + stream = std::make_unique( + data_part_info_for_read->getDataPartStorage(), MergeTreeDataPartCompact::DATA_FILE_NAME, + MergeTreeDataPartCompact::DATA_FILE_EXTENSION, data_part_info_for_read->getMarksCount(), + all_mark_ranges, stream_settings,uncompressed_cache, + data_part_info_for_read->getFileSizeOrZero(MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION), + marks_loader, profile_callback, clock_type); + + initialized = true; +} +catch (...) +{ + if (!isRetryableException(std::current_exception())) + data_part_info_for_read->reportBroken(); + throw; +} + +} diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.h b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.h new file mode 100644 index 00000000000..f358120e717 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.h @@ -0,0 +1,33 @@ +#pragma once +#include +#include + +namespace DB +{ + +/// Reader for compact parts, that uses one buffer for +/// all column and doesn't support parallel prefetch of columns. +/// It's suitable for compact parts with small size of stripe. +class MergeTreeReaderCompactSingleBuffer : public MergeTreeReaderCompact +{ +public: + template + explicit MergeTreeReaderCompactSingleBuffer(Args &&... args) + : MergeTreeReaderCompact{std::forward(args)...} + { + fillColumnPositions(); + } + + /// Returns the number of rows has been read or zero if there is no columns to read. + /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark + size_t readRows(size_t from_mark, size_t current_task_last_mark, + bool continue_reading, size_t max_rows_to_read, Columns & res_columns) override; + +private: + void init(); + + bool initialized = false; + std::unique_ptr stream; +}; + +} diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 8e460630d5e..40a16176c69 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -16,43 +16,28 @@ namespace ErrorCodes } MergeTreeReaderStream::MergeTreeReaderStream( - MergeTreeDataPartInfoForReaderPtr data_part_reader_, - const String & path_prefix_, - const String & data_file_extension_, - size_t marks_count_, - const MarkRanges & all_mark_ranges_, - const MergeTreeReaderSettings & settings_, - MarkCache * mark_cache_, - UncompressedCache * uncompressed_cache_, - size_t file_size_, - const MergeTreeIndexGranularityInfo * index_granularity_info_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_, - clockid_t clock_type_, - bool is_low_cardinality_dictionary_, - ThreadPool * load_marks_cache_threadpool_) - : settings(settings_) - , profile_callback(profile_callback_) + DataPartStoragePtr data_part_storage_, + const String & path_prefix_, + const String & data_file_extension_, + size_t marks_count_, + const MarkRanges & all_mark_ranges_, + const MergeTreeReaderSettings & settings_, + UncompressedCache * uncompressed_cache_, + size_t file_size_, + MergeTreeMarksLoaderPtr marks_loader_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, + clockid_t clock_type_) + : profile_callback(profile_callback_) , clock_type(clock_type_) , all_mark_ranges(all_mark_ranges_) - , file_size(file_size_) - , uncompressed_cache(uncompressed_cache_) - , data_part_storage(data_part_reader_->getDataPartStorage()) + , data_part_storage(std::move(data_part_storage_)) , path_prefix(path_prefix_) , data_file_extension(data_file_extension_) - , is_low_cardinality_dictionary(is_low_cardinality_dictionary_) + , uncompressed_cache(uncompressed_cache_) + , settings(settings_) , marks_count(marks_count_) - , mark_cache(mark_cache_) - , save_marks_in_cache(settings.save_marks_in_cache) - , index_granularity_info(index_granularity_info_) - , marks_loader( - data_part_reader_, - mark_cache, - index_granularity_info->getMarksFilePath(path_prefix), - marks_count, - *index_granularity_info, - save_marks_in_cache, - settings.read_settings, - load_marks_cache_threadpool_) + , file_size(file_size_) + , marks_loader(std::move(marks_loader_)) { } @@ -60,21 +45,12 @@ void MergeTreeReaderStream::init() { if (initialized) return; + initialized = true; + marks_getter = marks_loader->loadMarks(); + /// Compute the size of the buffer. - size_t max_mark_range_bytes = 0; - size_t sum_mark_range_bytes = 0; - - for (const auto & mark_range : all_mark_ranges) - { - size_t left_mark = mark_range.begin; - size_t right_mark = mark_range.end; - size_t left_offset = left_mark < marks_count ? marks_loader.getMark(left_mark).offset_in_compressed_file : 0; - auto mark_range_bytes = getRightOffset(right_mark) - left_offset; - - max_mark_range_bytes = std::max(max_mark_range_bytes, mark_range_bytes); - sum_mark_range_bytes += mark_range_bytes; - } + auto [max_mark_range_bytes, sum_mark_range_bytes] = estimateMarkRangeBytes(all_mark_ranges); std::optional estimated_sum_mark_range_bytes; if (sum_mark_range_bytes) @@ -83,7 +59,7 @@ void MergeTreeReaderStream::init() /// Avoid empty buffer. May happen while reading dictionary for DataTypeLowCardinality. /// For example: part has single dictionary and all marks point to the same position. ReadSettings read_settings = settings.read_settings; - if (max_mark_range_bytes != 0) + if (settings.adjust_read_buffer_size && max_mark_range_bytes != 0) read_settings = read_settings.adjustBufferSize(max_mark_range_bytes); //// Empty buffer does not makes progress. @@ -102,7 +78,8 @@ void MergeTreeReaderStream::init() read_settings, estimated_sum_mark_range_bytes, std::nullopt); }, - uncompressed_cache); + uncompressed_cache, + settings.allow_different_codecs); if (profile_callback) buffer->setProfileCallback(profile_callback, clock_type); @@ -121,7 +98,7 @@ void MergeTreeReaderStream::init() path_prefix + data_file_extension, read_settings, estimated_sum_mark_range_bytes, - std::nullopt)); + std::nullopt), settings.allow_different_codecs); if (profile_callback) buffer->setProfileCallback(profile_callback, clock_type); @@ -135,99 +112,10 @@ void MergeTreeReaderStream::init() } } - -size_t MergeTreeReaderStream::getRightOffset(size_t right_mark) -{ - /// NOTE: if we are reading the whole file, then right_mark == marks_count - /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. - - /// Special case, can happen in Collapsing/Replacing engines - if (marks_count == 0) - return 0; - - assert(right_mark <= marks_count); - - if (0 < right_mark && right_mark < marks_count) - { - /// Find the right border of the last mark we need to read. - /// To do that let's find the upper bound of the offset of the last - /// included mark. - - if (is_low_cardinality_dictionary) - { - - /// In LowCardinality dictionary several consecutive marks can point to the same offset. - /// - /// Also, in some cases, when one granule is not-atomically written (which is possible at merges) - /// one granule may require reading of two dictionaries which starts from different marks. - /// The only correct way is to take offset from at least next different granule from the right one. - /// So, that's why we have to read one extra granule to the right, - /// while reading dictionary of LowCardinality. - /// - /// Example: - /// Mark 0, points to [0, 8] - /// Mark 1, points to [0, 8] - /// Mark 2, points to [0, 8] - /// Mark 3, points to [0, 8] - /// Mark 4, points to [42336, 2255] - /// Mark 5, points to [42336, 2255] <--- for example need to read until 5 - /// Mark 6, points to [42336, 2255] <--- not suitable, because have same offset - /// Mark 7, points to [84995, 7738] <--- next different mark - /// Mark 8, points to [84995, 7738] - /// Mark 9, points to [126531, 8637] <--- what we are looking for - - auto indices = collections::range(right_mark, marks_count); - auto next_different_mark = [&](auto lhs, auto rhs) - { - return marks_loader.getMark(lhs).asTuple() < marks_loader.getMark(rhs).asTuple(); - }; - auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_mark)); - - if (it == indices.end()) - return file_size; - - right_mark = *it; - } - - /// This is a good scenario. The compressed block is finished within the right mark, - /// and previous mark was different. - if (marks_loader.getMark(right_mark).offset_in_decompressed_block == 0 - && marks_loader.getMark(right_mark) != marks_loader.getMark(right_mark - 1)) - return marks_loader.getMark(right_mark).offset_in_compressed_file; - - /// If right_mark has non-zero offset in decompressed block, we have to - /// read its compressed block in a whole, because it may consist of data from previous granule. - /// - /// For example: - /// Mark 6, points to [42336, 2255] - /// Mark 7, points to [84995, 7738] <--- right_mark - /// Mark 8, points to [84995, 7738] - /// Mark 9, points to [126531, 8637] <--- what we are looking for - /// - /// Since mark 7 starts from offset in decompressed block 7738, - /// it has some data from mark 6 and we have to read - /// compressed block [84995; 126531 in a whole. - - auto indices = collections::range(right_mark, marks_count); - auto next_different_compressed_offset = [&](auto lhs, auto rhs) - { - return marks_loader.getMark(lhs).offset_in_compressed_file < marks_loader.getMark(rhs).offset_in_compressed_file; - }; - auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_compressed_offset)); - - if (it != indices.end()) - return marks_loader.getMark(*it).offset_in_compressed_file; - } - else if (right_mark == 0) - return marks_loader.getMark(right_mark).offset_in_compressed_file; - - return file_size; -} - -void MergeTreeReaderStream::seekToMark(size_t index) +void MergeTreeReaderStream::seekToMarkAndColumn(size_t row_index, size_t column_position) { init(); - MarkInCompressedFile mark = marks_loader.getMark(index); + const auto & mark = marks_getter->getMark(row_index, column_position); try { @@ -237,7 +125,7 @@ void MergeTreeReaderStream::seekToMark(size_t index) { /// Better diagnostics. if (e.code() == ErrorCodes::ARGUMENT_OUT_OF_BOUND) - e.addMessage("(while seeking to mark " + toString(index) + e.addMessage("(while seeking to mark " + toString(row_index) + " of column " + path_prefix + "; offsets are: " + toString(mark.offset_in_compressed_file) + " " + toString(mark.offset_in_decompressed_block) + ")"); @@ -274,6 +162,7 @@ void MergeTreeReaderStream::adjustRightMark(size_t right_mark) */ init(); auto right_offset = getRightOffset(right_mark); + if (!right_offset) { if (last_right_offset && *last_right_offset == 0) @@ -304,4 +193,276 @@ CompressedReadBufferBase * MergeTreeReaderStream::getCompressedDataBuffer() return compressed_data_buffer; } +size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) const +{ + /// NOTE: if we are reading the whole file, then right_mark == marks_count + /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. + + /// Special case, can happen in Collapsing/Replacing engines + if (marks_count == 0) + return 0; + + assert(right_mark <= marks_count); + + if (right_mark == 0) + return marks_getter->getMark(right_mark, 0).offset_in_compressed_file; + + if (right_mark == marks_count) + return file_size; + + /// Find the right border of the last mark we need to read. + /// To do that let's find the upper bound of the offset of the last + /// included mark. + + if (settings.is_low_cardinality_dictionary) + { + /// In LowCardinality dictionary several consecutive marks can point to the same offset. + /// + /// Also, in some cases, when one granule is not-atomically written (which is possible at merges) + /// one granule may require reading of two dictionaries which starts from different marks. + /// The only correct way is to take offset from at least next different granule from the right one. + /// So, that's why we have to read one extra granule to the right, + /// while reading dictionary of LowCardinality. + /// + /// Example: + /// Mark 0, points to [0, 8] + /// Mark 1, points to [0, 8] + /// Mark 2, points to [0, 8] + /// Mark 3, points to [0, 8] + /// Mark 4, points to [42336, 2255] + /// Mark 5, points to [42336, 2255] <--- for example need to read until 5 + /// Mark 6, points to [42336, 2255] <--- not suitable, because have same offset + /// Mark 7, points to [84995, 7738] <--- next different mark + /// Mark 8, points to [84995, 7738] + /// Mark 9, points to [126531, 8637] <--- what we are looking for + + auto indices = collections::range(right_mark, marks_count); + auto next_different_mark = [&](auto lhs, auto rhs) + { + return marks_getter->getMark(lhs, 0).asTuple() < marks_getter->getMark(rhs, 0).asTuple(); + }; + + auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_mark)); + if (it == indices.end()) + return file_size; + + right_mark = *it; + } + + /// This is a good scenario. The compressed block is finished within the right mark, + /// and previous mark was different. + if (marks_getter->getMark(right_mark, 0).offset_in_decompressed_block == 0 + && marks_getter->getMark(right_mark, 0) != marks_getter->getMark(right_mark - 1, 0)) + return marks_getter->getMark(right_mark, 0).offset_in_compressed_file; + + /// If right_mark has non-zero offset in decompressed block, we have to + /// read its compressed block in a whole, because it may consist of data from previous granule. + /// + /// For example: + /// Mark 6, points to [42336, 2255] + /// Mark 7, points to [84995, 7738] <--- right_mark + /// Mark 8, points to [84995, 7738] + /// Mark 9, points to [126531, 8637] <--- what we are looking for + /// + /// Since mark 7 starts from offset in decompressed block 7738, + /// it has some data from mark 6 and we have to read + /// compressed block [84995; 126531 in a whole. + + auto indices = collections::range(right_mark, marks_count); + auto next_different_compressed_offset = [&](auto lhs, auto rhs) + { + return marks_getter->getMark(lhs, 0).offset_in_compressed_file < marks_getter->getMark(rhs, 0).offset_in_compressed_file; + }; + + auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_compressed_offset)); + if (it != indices.end()) + return marks_getter->getMark(*it, 0).offset_in_compressed_file; + + return file_size; +} + +std::pair MergeTreeReaderStreamSingleColumn::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +{ + assert(marks_getter != nullptr); + + size_t max_range_bytes = 0; + size_t sum_range_bytes = 0; + + for (const auto & mark_range : mark_ranges) + { + size_t left_mark = mark_range.begin; + size_t right_mark = mark_range.end; + size_t left_offset = left_mark < marks_count ? marks_getter->getMark(left_mark, 0).offset_in_compressed_file : 0; + auto mark_range_bytes = getRightOffset(right_mark) - left_offset; + + max_range_bytes = std::max(max_range_bytes, mark_range_bytes); + sum_range_bytes += mark_range_bytes; + } + + return {max_range_bytes, sum_range_bytes}; +} + +size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) const +{ + /// NOTE: if we are reading the whole file, then right_mark == marks_count + /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. + + /// Special case, can happen in Collapsing/Replacing engines + if (marks_count == 0) + return 0; + + assert(right_mark_non_included <= marks_count); + + if (right_mark_non_included == 0) + return marks_getter->getMark(right_mark_non_included, column_position).offset_in_compressed_file; + + size_t right_mark_included = right_mark_non_included - 1; + if (right_mark_non_included != marks_count + && marks_getter->getMark(right_mark_non_included, column_position).offset_in_decompressed_block != 0) + ++right_mark_included; + + /// The right bound for case, where there is no smaller suitable mark + /// is the start of the next stripe (in which the next column is written) + /// because each stripe always start from a new compressed block. + const auto & right_mark_in_file = marks_getter->getMark(right_mark_included, column_position); + auto next_stripe_right_mark_in_file = getStartOfNextStripeMark(right_mark_included, column_position); + + /// Try to find suitable right mark in current stripe. + for (size_t mark = right_mark_included + 1; mark < marks_count; ++mark) + { + const auto & current_mark = marks_getter->getMark(mark, column_position); + /// We found first mark that starts from the new compressed block. + if (current_mark.offset_in_compressed_file > right_mark_in_file.offset_in_compressed_file) + { + /// If it is in current stripe return it to reduce amount of read data. + if (current_mark < next_stripe_right_mark_in_file) + return current_mark.offset_in_compressed_file; + + /// Otherwise return start of new stripe as an upper bound. + break; + } + } + + return next_stripe_right_mark_in_file.offset_in_compressed_file; +} + +std::pair +MergeTreeReaderStreamMultipleColumns::estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) const +{ + assert(marks_getter != nullptr); + + /// As a maximal range we return the maximal size of a whole stripe. + size_t max_range_bytes = 0; + size_t sum_range_bytes = 0; + + for (const auto & mark_range : mark_ranges) + { + auto start_of_stripe_mark = marks_getter->getMark(mark_range.begin, column_position); + auto start_of_next_stripe_mark = getStartOfNextStripeMark(mark_range.begin, column_position); + + for (size_t mark = mark_range.begin; mark < mark_range.end; ++mark) + { + const auto & current_mark = marks_getter->getMark(mark, column_position); + + /// We found a start of new stripe, now update values. + if (current_mark > start_of_next_stripe_mark) + { + auto current_range_bytes = getRightOffsetOneColumn(mark, column_position) - start_of_stripe_mark.offset_in_compressed_file; + + max_range_bytes = std::max(max_range_bytes, current_range_bytes); + sum_range_bytes += current_range_bytes; + + start_of_stripe_mark = current_mark; + start_of_next_stripe_mark = getStartOfNextStripeMark(mark, column_position); + } + } + + auto current_range_bytes = getRightOffsetOneColumn(mark_range.end, column_position) - start_of_stripe_mark.offset_in_compressed_file; + + max_range_bytes = std::max(max_range_bytes, current_range_bytes); + sum_range_bytes += current_range_bytes; + } + + return {max_range_bytes, sum_range_bytes}; +} + +MarkInCompressedFile MergeTreeReaderStreamMultipleColumns::getStartOfNextStripeMark(size_t row_index, size_t column_position) const +{ + const auto & current_mark = marks_getter->getMark(row_index, column_position); + + if (marks_getter->getNumColumns() == 1) + return MarkInCompressedFile{file_size, 0}; + + if (column_position + 1 == marks_getter->getNumColumns()) + { + /** + * In case of the last column (c3), we have the following picture: + * c1 c2 c3 + * x x x + * (row_index, 0) -> o x o <- (row_index, column_position) + * x x x + * ------- <- start of new stripe + * what we are -> o x x + * looking for x x x + * x x x + * ------- + * So, we need to iterate forward. + */ + size_t mark_index = row_index + 1; + while (mark_index < marks_count && marks_getter->getMark(mark_index, 0) <= current_mark) + ++mark_index; + + return mark_index == marks_count + ? MarkInCompressedFile{file_size, 0} + : marks_getter->getMark(mark_index, 0); + } + + /** + * Otherwise, we have the following picture: + * c1 c2 c3 + * x x o <- what we are looking for + * (row, column) --> o o <- (row, column + 1) + * x x x + * ------- <- start of new stripe + * So, we need to iterate backward. + */ + + ssize_t mark_index = row_index; + while (mark_index >= 0 && marks_getter->getMark(mark_index, column_position + 1) >= current_mark) + --mark_index; + + return marks_getter->getMark(mark_index + 1, column_position + 1); +} + +size_t MergeTreeReaderStreamOneOfMultipleColumns::getRightOffset(size_t right_mark_non_included) const +{ + return getRightOffsetOneColumn(right_mark_non_included, column_position); +} + +std::pair MergeTreeReaderStreamOneOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +{ + return estimateMarkRangeBytesOneColumn(mark_ranges, column_position); +} + +size_t MergeTreeReaderStreamAllOfMultipleColumns::getRightOffset(size_t right_mark_non_included) const +{ + return getRightOffsetOneColumn(right_mark_non_included, marks_loader->getNumColumns() - 1); +} + +std::pair MergeTreeReaderStreamAllOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +{ + size_t max_range_bytes = 0; + size_t sum_range_bytes = 0; + + for (size_t i = 0; i < marks_getter->getNumColumns(); ++i) + { + auto [current_max, current_sum] = estimateMarkRangeBytesOneColumn(mark_ranges, i); + + max_range_bytes = std::max(max_range_bytes, current_max); + sum_range_bytes += current_sum; + } + + return {max_range_bytes, sum_range_bytes}; +} + } diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index 49ce3103434..f3ca6953ceb 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -14,27 +14,31 @@ namespace DB { -/// Class for reading a single column (or index). +/// Basic and the most low-level class +/// for reading single columns or indexes. class MergeTreeReaderStream { public: MergeTreeReaderStream( - MergeTreeDataPartInfoForReaderPtr data_part_reader_, + DataPartStoragePtr data_part_storage_, const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, - const MarkRanges & all_mark_ranges, + const MarkRanges & all_mark_ranges_, const MergeTreeReaderSettings & settings_, - MarkCache * mark_cache, - UncompressedCache * uncompressed_cache, + UncompressedCache * uncompressed_cache_, size_t file_size_, - const MergeTreeIndexGranularityInfo * index_granularity_info_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback, - clockid_t clock_type, - bool is_low_cardinality_dictionary_, - ThreadPool * load_marks_cache_threadpool_); + MergeTreeMarksLoaderPtr marks_loader_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, + clockid_t clock_type_); - void seekToMark(size_t index); + virtual ~MergeTreeReaderStream() = default; + + /// Seeks to start of @row_index mark. Column position is implementation defined. + virtual void seekToMark(size_t row_index) = 0; + + /// Seeks to exact mark in file. + void seekToMarkAndColumn(size_t row_index, size_t column_position); void seekToStart(); @@ -48,39 +52,111 @@ public: CompressedReadBufferBase * getCompressedDataBuffer(); private: - void init(); - size_t getRightOffset(size_t right_mark); + /// Returns offset in file up to which it's needed to read file to read all rows up to @right_mark mark. + virtual size_t getRightOffset(size_t right_mark) const = 0; + + /// Returns estimated max amount of bytes to read among mark ranges (which is used as size for read buffer) + /// and total amount of bytes to read in all mark ranges. + virtual std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const = 0; - const MergeTreeReaderSettings settings; const ReadBufferFromFileBase::ProfileCallback profile_callback; - clockid_t clock_type; + const clockid_t clock_type; const MarkRanges all_mark_ranges; - size_t file_size; - UncompressedCache * uncompressed_cache; - DataPartStoragePtr data_part_storage; - std::string path_prefix; - std::string data_file_extension; - - bool is_low_cardinality_dictionary = false; - - size_t marks_count; + const DataPartStoragePtr data_part_storage; + const std::string path_prefix; + const std::string data_file_extension; + UncompressedCache * const uncompressed_cache; ReadBuffer * data_buffer; CompressedReadBufferBase * compressed_data_buffer; - MarkCache * mark_cache; - bool save_marks_in_cache; + bool initialized = false; - std::optional last_right_offset; - const MergeTreeIndexGranularityInfo * index_granularity_info; - std::unique_ptr cached_buffer; std::unique_ptr non_cached_buffer; - MergeTreeMarksLoader marks_loader; +protected: + void init(); + + const MergeTreeReaderSettings settings; + const size_t marks_count; + const size_t file_size; + + const MergeTreeMarksLoaderPtr marks_loader; + MergeTreeMarksGetterPtr marks_getter; +}; + +/// Class for reading a single column (or index) from file +/// that contains a single column (for wide parts). +class MergeTreeReaderStreamSingleColumn : public MergeTreeReaderStream +{ +public: + template + explicit MergeTreeReaderStreamSingleColumn(Args &&... args) + : MergeTreeReaderStream{std::forward(args)...} + { + } + + size_t getRightOffset(size_t right_mark_non_included) const override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, 0); } +}; + +/// Base class for reading from file that contains multiple columns. +/// It is used to read from compact parts. +/// See more details about data layout in MergeTreeDataPartCompact.h. +class MergeTreeReaderStreamMultipleColumns : public MergeTreeReaderStream +{ +public: + template + explicit MergeTreeReaderStreamMultipleColumns(Args &&... args) + : MergeTreeReaderStream{std::forward(args)...} + { + } + +protected: + size_t getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) const; + std::pair estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) const; + MarkInCompressedFile getStartOfNextStripeMark(size_t row_index, size_t column_position) const; +}; + +/// Class for reading a single column from file that contains multiple columns +/// (for parallel reading from compact parts with large stripes). +class MergeTreeReaderStreamOneOfMultipleColumns : public MergeTreeReaderStreamMultipleColumns +{ +public: + template + explicit MergeTreeReaderStreamOneOfMultipleColumns(size_t column_position_, Args &&... args) + : MergeTreeReaderStreamMultipleColumns{std::forward(args)...} + , column_position(column_position_) + { + } + + size_t getRightOffset(size_t right_mark_non_included) const override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, column_position); } + +private: + const size_t column_position; +}; + +/// Class for reading multiple columns from file that contains multiple columns +/// (for reading from compact parts with small stripes). +class MergeTreeReaderStreamAllOfMultipleColumns : public MergeTreeReaderStreamMultipleColumns +{ +public: + template + explicit MergeTreeReaderStreamAllOfMultipleColumns(Args &&... args) + : MergeTreeReaderStreamMultipleColumns{std::forward(args)...} + { + } + + size_t getRightOffset(size_t right_mark_non_included) const override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, 0); } }; } diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index d34a58a25b0..394a22835f1 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -225,18 +225,29 @@ void MergeTreeReaderWide::addStreams( return; } - has_any_stream = true; - bool is_lc_dict = substream_path.size() > 1 && substream_path[substream_path.size() - 2].type == ISerialization::Substream::Type::DictionaryKeys; - auto context = data_part_info_for_read->getContext(); auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; - streams.emplace(*stream_name, std::make_unique( - data_part_info_for_read, *stream_name, DATA_FILE_EXTENSION, - data_part_info_for_read->getMarksCount(), all_mark_ranges, settings, mark_cache, + auto marks_loader = std::make_shared( + data_part_info_for_read, + mark_cache, + data_part_info_for_read->getIndexGranularityInfo().getMarksFilePath(*stream_name), + data_part_info_for_read->getMarksCount(), + data_part_info_for_read->getIndexGranularityInfo(), + settings.save_marks_in_cache, + settings.read_settings, + load_marks_threadpool, + /*num_columns_in_mark=*/ 1); + + has_any_stream = true; + auto stream_settings = settings; + stream_settings.is_low_cardinality_dictionary = substream_path.size() > 1 && substream_path[substream_path.size() - 2].type == ISerialization::Substream::Type::DictionaryKeys; + + streams.emplace(*stream_name, std::make_unique( + data_part_info_for_read->getDataPartStorage(), *stream_name, DATA_FILE_EXTENSION, + data_part_info_for_read->getMarksCount(), all_mark_ranges, stream_settings, uncompressed_cache, data_part_info_for_read->getFileSizeOrZero(*stream_name + DATA_FILE_EXTENSION), - &data_part_info_for_read->getIndexGranularityInfo(), - profile_callback, clock_type, is_lc_dict, load_marks_threadpool)); + std::move(marks_loader), profile_callback, clock_type)); }; serialization->enumerateStreams(callback); diff --git a/src/Storages/StorageMergeTreeIndex.cpp b/src/Storages/StorageMergeTreeIndex.cpp index 5f89849e920..329275f4605 100644 --- a/src/Storages/StorageMergeTreeIndex.cpp +++ b/src/Storages/StorageMergeTreeIndex.cpp @@ -184,9 +184,11 @@ private: auto & compressed_data = compressed->getData(); auto & uncompressed_data = uncompressed->getData(); + auto marks_getter = marks_loader->loadMarks(); + for (size_t i = 0; i < num_rows; ++i) { - auto mark = marks_loader->getMark(i, col_idx); + auto mark = marks_getter->getMark(i, col_idx); compressed_data[i] = mark.offset_in_compressed_file; uncompressed_data[i] = mark.offset_in_decompressed_block; From 265d91747fceb596d76ea7422f6dfa6995b61870 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:03:12 +0000 Subject: [PATCH 0187/1165] Remove log debug --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 62 +++++-------------- src/Storages/System/StorageSystemNumbers.cpp | 2 - src/Storages/System/attachSystemTables.cpp | 1 + .../TableFunctionGenerateSeries.cpp | 23 ++++--- src/TableFunctions/TableFunctionNumbers.cpp | 11 ++-- 5 files changed, 35 insertions(+), 64 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 52b5eeb2ca3..503dd534d89 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -29,6 +29,15 @@ extern const int TOO_MANY_ROWS; namespace { +template +inline void iota_with_step_optimized(T * begin, size_t count, T first_value, T step) +{ + if (step == 1) + iota(begin, count, first_value); + else + iota_with_step(begin, count, first_value, step); +} + class NumbersSource : public ISource { public: @@ -66,14 +75,8 @@ protected: UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * current_end = &vec[real_block_size]; - if (step == 1) - { - iota(pos, static_cast(current_end - pos), curr); - } - else - { - iota_with_step(pos, static_cast(current_end - pos), curr, step); - } + + iota_with_step_optimized(pos, static_cast(current_end - pos), curr, step); next += chunk_step; @@ -101,8 +104,6 @@ using RangesWithStep = std::vector; std::optional stepped_range_from_range(const Range & r, UInt64 step, UInt64 remainder) { - // LOG_DEBUG(&Poco::Logger::get("Stepped from range"), - // "stepped from range"); if ((r.right.get() == 0) && (!r.right_included)) return std::nullopt; UInt64 begin = (r.left.get() / step) * step; @@ -155,7 +156,7 @@ public: using RangesStatePtr = std::shared_ptr; - [[maybe_unused]] NumbersRangedSource( + NumbersRangedSource( const RangesWithStep & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, @@ -167,17 +168,6 @@ public: , base_block_size(base_block_size_) , step(step_) { - // for (const auto& range_with_step : ranges_) { - // // LOG_DEBUG(&Poco::Logger::get("Ranges With Step"), - // // "Ranges: {} {} {} {} {}", - // // range_with_step.range.left.get(), - // // range_with_step.range.right.get(), - // // range_with_step.range.left_included, - // // range_with_step.range.right_included, - // // range_with_step.step); - // // LOG_DEBUG(&Poco::Logger::get("Ranges With Step"), - // // "Step: {}", step); - // } } String getName() const override { return "NumbersRange"; } @@ -242,8 +232,6 @@ protected: RangesPos start, end; auto block_size = findRanges(start, end, base_block_size); - // LOG_DEBUG(&Poco::Logger::get("Found range"), "Evth: {} {} {} {} {} {}", start.offset_in_ranges, static_cast(start.offset_in_range), end.offset_in_ranges, static_cast(end.offset_in_range), base_block_size, block_size); - if (!block_size) return {}; @@ -259,11 +247,6 @@ protected: while (block_size - provided != 0) { UInt64 need = block_size - provided; - // LOG_DEBUG(&Poco::Logger::get("Indices:"), - // "Indices: {} {}, provided: {}", - // ranges.size(), - // cursor.offset_in_ranges, - // provided); auto & range = ranges[cursor.offset_in_ranges]; UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges @@ -286,14 +269,7 @@ protected: auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = (end_value_64 - start_value_64) / this->step; - if (step == 1) - { - iota(pos, static_cast(size), start_value_64); - } - else - { - iota_with_step(pos, static_cast(size), start_value_64, step); - } + iota_with_step_optimized(pos, static_cast(size), start_value_64, step); pos += size; } }; @@ -302,14 +278,7 @@ protected: { UInt64 start_value = range.left + cursor.offset_in_range * step; /// end_value will never overflow - if (step == 1) - { - iota(pos, static_cast(need), start_value); - } - else - { - iota_with_step(pos, static_cast(need), start_value, step); - } + iota_with_step_optimized(pos, static_cast(need), start_value, step); pos += need; provided += need; cursor.offset_in_range += need; @@ -467,9 +436,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() Pipe pipe; Ranges ranges; - - // LOG_DEBUG(&Poco::Logger::get("parameters"), "Parameters: {} {} {} {}", numbers_storage.step, numbers_storage.offset, numbers_storage.limit.has_value(), numbers_storage.limit.has_value() ? numbers_storage.limit.value() : UInt64{0}); - if (numbers_storage.limit.has_value() && (numbers_storage.limit.value() == 0)) { pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index d47c592c4b3..32d16ef38f9 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -40,9 +40,7 @@ void StorageSystemNumbers::read( size_t max_block_size, size_t num_streams) { - // LOG_DEBUG(&Poco::Logger::get("Reading from SystemNumbers"), "Limit : {}", limit.value()); query_plan.addStep(std::make_unique( column_names, query_info, storage_snapshot, context, shared_from_this(), max_block_size, num_streams)); } - } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 3fa55fe28c5..ada812c9f54 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -122,6 +122,7 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "number"); attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "number"); attachNoDescription(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); + attachNoDescription(context, system_database, "generateSeries", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); attachNoDescription(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attachNoDescription(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); attach(context, system_database, "databases", "Lists all databases of the current server."); diff --git a/src/TableFunctions/TableFunctionGenerateSeries.cpp b/src/TableFunctions/TableFunctionGenerateSeries.cpp index 65c4c4915c2..ee833250d0b 100644 --- a/src/TableFunctions/TableFunctionGenerateSeries.cpp +++ b/src/TableFunctions/TableFunctionGenerateSeries.cpp @@ -24,14 +24,14 @@ namespace ErrorCodes namespace { -/* numbers(limit), numbers_mt(limit) - * - the same as SELECT number FROM system.numbers LIMIT limit. - * Used for testing purposes, as a simple example of table function. - */ +constexpr std::array names = {"generate_series", "generateSeries"}; + +template class TableFunctionGenerateSeries : public ITableFunction { public: - static constexpr auto name = "generate_series"; + static_assert(alias_num < names.size()); + static constexpr auto name = names[alias_num]; std::string getName() const override { return name; } bool hasStaticStructure() const override { return true; } @@ -49,13 +49,15 @@ private: ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; }; -ColumnsDescription TableFunctionGenerateSeries::getActualTableStructure(ContextPtr /*context*/, bool /*is_insert_query*/) const +template +ColumnsDescription TableFunctionGenerateSeries::getActualTableStructure(ContextPtr /*context*/, bool /*is_insert_query*/) const { /// NOTE: https://bugs.llvm.org/show_bug.cgi?id=47418 return ColumnsDescription{{{"generate_series", std::make_shared()}}}; } -StoragePtr TableFunctionGenerateSeries::executeImpl( +template +StoragePtr TableFunctionGenerateSeries::executeImpl( const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, @@ -86,7 +88,8 @@ StoragePtr TableFunctionGenerateSeries::executeImpl( throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'limit' or 'offset, limit'.", getName()); } -UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr & argument) const +template +UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr & argument) const { const auto & [field, type] = evaluateConstantExpression(argument, context); @@ -108,8 +111,8 @@ UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr void registerTableFunctionGenerateSeries(TableFunctionFactory & factory) { - factory.registerFunction({.documentation = {}, .allow_readonly = true}); - // factory.registerFunction({.documentation = {}, .allow_readonly = true}); + factory.registerFunction>({.documentation = {}, .allow_readonly = true}); + factory.registerFunction>({.documentation = {}, .allow_readonly = true}); } } diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index bcda8dc6a5e..2989eb5fbef 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include "base/types.h" #include "registerTableFunctions.h" @@ -68,15 +70,16 @@ StoragePtr TableFunctionNumbers::executeImpl( { auto arguments = function->arguments->children; - if (arguments.size() != 1 && arguments.size() != 2) + if ((arguments.empty()) || (arguments.size() >= 4)) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); - UInt64 offset = arguments.size() == 2 ? evaluateArgument(context, arguments[0]) : 0; - UInt64 length = arguments.size() == 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); + UInt64 offset = arguments.size() >= 2 ? evaluateArgument(context, arguments[0]) : 0; + UInt64 length = arguments.size() >= 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); + UInt64 step = arguments.size() == 3 ? evaluateArgument(context, arguments[2]) : 1; auto res = std::make_shared( - StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset); + StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset, step); res->startup(); return res; } From d0ed47e655913f66fec5ce31dbf10a2e045424ef Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:04:31 +0000 Subject: [PATCH 0188/1165] Return right cmake settings --- CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6847b1bf994..8c4e16eace2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,19 +54,19 @@ endif () # ccache ignore it. option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF) if (ENABLE_CHECK_HEAVY_BUILDS) - # # set DATA (since RSS does not work since 2.6.x+) to 5G - # set (RLIMIT_DATA 5000000000) - # # set VIRT (RLIMIT_AS) to 10G (DATA*2) - # set (RLIMIT_AS 10000000000) - # # set CPU time limit to 1000 seconds - # set (RLIMIT_CPU 1000) + # set DATA (since RSS does not work since 2.6.x+) to 5G + set (RLIMIT_DATA 5000000000) + # set VIRT (RLIMIT_AS) to 10G (DATA*2) + set (RLIMIT_AS 10000000000) + # set CPU time limit to 1000 seconds + set (RLIMIT_CPU 1000) - # # -fsanitize=memory and address are too heavy - # if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE) - # set (RLIMIT_DATA 10000000000) # 10G - # endif() + # -fsanitize=memory and address are too heavy + if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE) + set (RLIMIT_DATA 10000000000) # 10G + endif() - # set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) + set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) endif () if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") From 13b27a8d3663c6c160b826eef131601f226afacd Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 18 Mar 2024 16:13:02 +0000 Subject: [PATCH 0189/1165] refactoring of reading from compact parts --- .../MergeTree/MergeTreeReaderCompact.cpp | 55 ++++++++++--------- .../MergeTree/MergeTreeReaderCompact.h | 3 +- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 3835c1e2054..a22bff6b8d2 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -78,31 +78,7 @@ void MergeTreeReaderCompact::fillColumnPositions() /// we have to read its offsets if they exist. if (!position && is_array) { - NameAndTypePair column_to_read_with_subcolumns = column_to_read; - auto [name_in_storage, subcolumn_name] = Nested::splitName(column_to_read.name); - - /// If it is a part of Nested, we need to get the column from - /// storage metadata which is converted to Nested type with subcolumns. - /// It is required for proper counting of shared streams. - if (!subcolumn_name.empty()) - { - /// If column is renamed get the new name from storage metadata. - if (alter_conversions->columnHasNewName(name_in_storage)) - name_in_storage = alter_conversions->getColumnNewName(name_in_storage); - - if (!storage_columns_with_collected_nested) - { - auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); - auto storage_columns_list = Nested::collect(storage_snapshot->getColumns(options)); - storage_columns_with_collected_nested = ColumnsDescription(std::move(storage_columns_list)); - } - - column_to_read_with_subcolumns = storage_columns_with_collected_nested - ->getColumnOrSubcolumn( - GetColumnsOptions::All, - Nested::concatenateName(name_in_storage, subcolumn_name)); - } - + auto column_to_read_with_subcolumns = getColumnConvertedToSubcolumnOfNested(column_to_read); auto name_level_for_offsets = findColumnForOffsets(column_to_read_with_subcolumns); if (name_level_for_offsets.has_value()) @@ -119,6 +95,35 @@ void MergeTreeReaderCompact::fillColumnPositions() } } +NameAndTypePair MergeTreeReaderCompact::getColumnConvertedToSubcolumnOfNested(const NameAndTypePair & column) +{ + if (!isArray(column.type)) + return column; + + /// If it is a part of Nested, we need to get the column from + /// storage metadata which is converted to Nested type with subcolumns. + /// It is required for proper counting of shared streams. + auto [name_in_storage, subcolumn_name] = Nested::splitName(column.name); + + if (subcolumn_name.empty()) + return column; + + /// If column is renamed get the new name from storage metadata. + if (alter_conversions->columnHasNewName(name_in_storage)) + name_in_storage = alter_conversions->getColumnNewName(name_in_storage); + + if (!storage_columns_with_collected_nested) + { + auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); + auto storage_columns_list = Nested::collect(storage_snapshot->getColumns(options)); + storage_columns_with_collected_nested = ColumnsDescription(std::move(storage_columns_list)); + } + + return storage_columns_with_collected_nested->getColumnOrSubcolumn( + GetColumnsOptions::All, + Nested::concatenateName(name_in_storage, subcolumn_name)); +} + void MergeTreeReaderCompact::readData( const NameAndTypePair & name_and_type, ColumnPtr & column, diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 9c66b4691a3..a783e595af5 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -31,10 +31,11 @@ public: const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_); - bool canReadIncompleteGranules() const override final { return false; } + bool canReadIncompleteGranules() const final { return false; } protected: void fillColumnPositions(); + NameAndTypePair getColumnConvertedToSubcolumnOfNested(const NameAndTypePair & column); using InputStreamGetter = ISerialization::InputStreamGetter; From 3fd5c6920579d927ee994646115211cae046c2e6 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:17:54 +0000 Subject: [PATCH 0190/1165] Add docs --- docs/en/sql-reference/table-functions/numbers.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/en/sql-reference/table-functions/numbers.md b/docs/en/sql-reference/table-functions/numbers.md index 7d3437b7d53..e0ff19b9824 100644 --- a/docs/en/sql-reference/table-functions/numbers.md +++ b/docs/en/sql-reference/table-functions/numbers.md @@ -8,6 +8,7 @@ sidebar_label: numbers `numbers(N)` – Returns a table with the single ‘number’ column (UInt64) that contains integers from 0 to N-1. `numbers(N, M)` - Returns a table with the single ‘number’ column (UInt64) that contains integers from N to (N + M - 1). +`numbers(N, M, S)` - Returns a table with the single ‘number’ column (UInt64) that contains integers from N to (N + M - 1) with step S. Similar to the `system.numbers` table, it can be used for testing and generating successive values, `numbers(N, M)` more efficient than `system.numbers`. @@ -21,6 +22,15 @@ SELECT * FROM system.numbers WHERE number BETWEEN 0 AND 9; SELECT * FROM system.numbers WHERE number IN (0, 1, 2, 3, 4, 5, 6, 7, 8, 9); ``` +And the following queries are equivalent: + +``` sql +SELECT number * 2 FROM numbers(10); +SELECT (number - 10) * 2 FROM numbers(10, 10); +SELECT * FROM numbers(0, 20, 2); +``` + + Examples: ``` sql From 2061fb25c5e88766bdce1f5a2ea9f0a734edbcf9 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:21:08 +0000 Subject: [PATCH 0191/1165] Add alias to doc --- docs/en/sql-reference/table-functions/generate_series.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/table-functions/generate_series.md b/docs/en/sql-reference/table-functions/generate_series.md index c5d29369627..9370cea8580 100644 --- a/docs/en/sql-reference/table-functions/generate_series.md +++ b/docs/en/sql-reference/table-functions/generate_series.md @@ -22,4 +22,7 @@ And the following queries return tables with the same content but different colu ``` sql SELECT * FROM numbers(10, 11) WHERE number % 3 == (10 % 3); SELECT * FROM generate_series(10, 20, 3) ; -``` \ No newline at end of file +``` + +### Alias +generateSeries \ No newline at end of file From 31d5049f803074989792e01d459640df3ab03c50 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 18 Mar 2024 16:26:05 +0000 Subject: [PATCH 0192/1165] Move logic of ignoring drop queries to server --- programs/client/Client.cpp | 7 +-- src/Client/ClientBase.cpp | 61 ------------------- src/Client/ClientBase.h | 8 --- src/Core/Settings.h | 1 + src/Interpreters/InterpreterDropQuery.cpp | 11 +++- src/Interpreters/InterpreterDropQuery.h | 2 +- tests/ci/stress.py | 4 +- tests/clickhouse-test | 2 +- ..._ignore_drop_queries_probability.reference | 1 + .../03013_ignore_drop_queries_probability.sql | 18 ++++++ 10 files changed, 36 insertions(+), 79 deletions(-) create mode 100644 tests/queries/0_stateless/03013_ignore_drop_queries_probability.reference create mode 100644 tests/queries/0_stateless/03013_ignore_drop_queries_probability.sql diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index caf2ce2f68a..a8a15245821 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -953,9 +953,8 @@ void Client::addOptions(OptionsDescription & options_description) ("opentelemetry-tracestate", po::value(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation") ("no-warnings", "disable warnings when client connects to server") - /// TODO: Left for compatibility as it's used in upgrade check, remove after next release and use ignore-drop-queries-probability + /// TODO: Left for compatibility as it's used in upgrade check, remove after next release and use server setting ignore_drop_queries_probability ("fake-drop", "Ignore all DROP queries, should be used only for testing") - ("ignore-drop-queries-probability", po::value(), "With specified probability ignore all DROP queries (replace them to TRUNCATE for engines like Memory/JOIN), should be used only for testing") ("accept-invalid-certificate", "Ignore certificate verification errors, equal to config parameters openSSL.client.invalidCertificateHandler.name=AcceptCertificateHandler and openSSL.client.verificationMode=none") ; @@ -1098,9 +1097,7 @@ void Client::processOptions(const OptionsDescription & options_description, if (options.count("no-warnings")) config().setBool("no-warnings", true); if (options.count("fake-drop")) - ignore_drop_queries_probability = 1; - if (options.count("ignore-drop-queries-probability")) - ignore_drop_queries_probability = std::min(options["ignore-drop-queries-probability"].as(), 1.); + config().setString("ignore_drop_queries_probability", "1"); if (options.count("accept-invalid-certificate")) { config().setString("openSSL.client.invalidCertificateHandler.name", "AcceptCertificateHandler"); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 9126205b868..de995ffe408 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -876,69 +876,8 @@ void ClientBase::processTextAsSingleQuery(const String & full_query) processError(full_query); } -String ClientBase::getTableEngine(const String & database, const String & table) -{ - auto is_interactive_copy = is_interactive; - auto format_copy = format; - - is_interactive = false; - format = "TSVRaw"; - String result; - output_format_buffer = std::make_unique(result); - String query; - if (database.empty()) - query = fmt::format("SELECT engine FROM system.tables where name='{}' and database=currentDatabase()", table); - else - query = fmt::format("SELECT engine FROM system.tables where name='{}' and database='{}'", table, database); - - try - { - processTextAsSingleQuery(query); - } - catch (...) - { - result = ""; - } - - output_format_buffer->finalize(); - output_format_buffer.reset(); - is_interactive = is_interactive_copy; - format = format_copy; - boost::trim(result); - return result; -} - -void ClientBase::ignoreDropQueryOrTruncateTable(const DB::ASTDropQuery * drop_query) -{ - const auto & database = drop_query->getDatabase(); - const auto & table = drop_query->getTable(); - /// Use TRUNCATE for Memory/JOIN table engines to reduce memory usage in tests. - String table_engine = getTableEngine(database, table); - if (table_engine == "Memory" || table_engine == "JOIN") - { - String truncate_query; - if (database.empty()) - truncate_query = fmt::format("TRUNCATE TABLE {}", table); - else - truncate_query = fmt::format("TRUNCATE TABLE {}.{}", database, table); - - auto is_interactive_copy = is_interactive; - is_interactive = false; - processTextAsSingleQuery(truncate_query); - is_interactive = is_interactive_copy; - } -} - void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr parsed_query) { - /// In tests we can ignore DROP queries with some probability. - const auto * drop_query = parsed_query->as(); - if (ignore_drop_queries_probability != 0 && drop_query && drop_query->kind == ASTDropQuery::Kind::Drop && std::uniform_real_distribution<>(0.0, 1.0)(thread_local_rng) <= ignore_drop_queries_probability) - { - ignoreDropQueryOrTruncateTable(drop_query); - return; - } - auto query = query_to_execute; /// Rewrite query only when we have query parameters. diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 462899994a2..74d065bf342 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -178,12 +178,6 @@ private: void initQueryIdFormats(); bool addMergeTreeSettings(ASTCreateQuery & ast_create); - void ignoreDropQueryOrTruncateTable(const ASTDropQuery * drop_query); - /// Request table engine from system.tables from server. - String getTableEngine(const String & database, const String & table); - /// Send TRUNCATE query for specific table. - void truncateTable(const String & database, const String & table); - protected: static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context); bool processMultiQueryFromFile(const String & file_name); @@ -314,8 +308,6 @@ protected: QueryProcessingStage::Enum query_processing_stage; ClientInfo::QueryKind query_kind; - double ignore_drop_queries_probability = 0; - struct HostAndPort { String host; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 415063eee84..15c1719833c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -857,6 +857,7 @@ class IColumn; M(Bool, optimize_uniq_to_count, true, "Rewrite uniq and its variants(except uniqUpTo) to count if subquery has distinct or group by clause.", 0) \ M(Bool, use_variant_as_common_type, false, "Use Variant as a result type for if/multiIf in case when there is no common type for arguments", 0) \ M(Bool, enable_order_by_all, true, "Enable sorting expression ORDER BY ALL.", 0) \ + M(Float, ignore_drop_queries_probability, 0, "If enabled, server will ignore all DROP table queries with specified probability (for Memory and JOIN engines it will replcase DROP to TRUNCATE). Used for testing purposes", 0) \ \ /** Experimental functions */ \ M(Bool, allow_experimental_materialized_postgresql_table, false, "Allows to use the MaterializedPostgreSQL table engine. Disabled by default, because this feature is experimental", 0) \ diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 72aa4cc63e3..fef1d215d0a 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -104,7 +104,7 @@ BlockIO InterpreterDropQuery::executeToTable(ASTDropQuery & query) return res; } -BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait) +BlockIO InterpreterDropQuery::executeToTableImpl(const ContextPtr & context_, ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait) { /// NOTE: it does not contain UUID, we will resolve it with locked DDLGuard auto table_id = StorageID(query); @@ -151,6 +151,15 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue "Table {} is not a Dictionary", table_id.getNameForLogs()); + if (settings.ignore_drop_queries_probability != 0 && ast_drop_query.kind == ASTDropQuery::Kind::Drop && std::uniform_real_distribution<>(0.0, 1.0)(thread_local_rng) <= settings.ignore_drop_queries_probability) + { + ast_drop_query.sync = false; + if (table->getName() != "Memory" && table->getName() != "Join") + return {}; + + ast_drop_query.kind = ASTDropQuery::Truncate; + } + /// Now get UUID, so we can wait for table data to be finally dropped table_id.uuid = database->tryGetTableUUID(table_id.table_name); diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 7ae544a7356..8829fbe9ea5 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -37,7 +37,7 @@ private: BlockIO executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait); BlockIO executeToTable(ASTDropQuery & query); - BlockIO executeToTableImpl(ContextPtr context_, ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait); + BlockIO executeToTableImpl(const ContextPtr& context_, ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait); static void waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait); diff --git a/tests/ci/stress.py b/tests/ci/stress.py index e04f06c951b..5ed88c8df7e 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -66,10 +66,10 @@ def get_options(i: int, upgrade_check: bool) -> str: if random.random() < 0.3: client_options.append(f"http_make_head_request={random.randint(0, 1)}") - # TODO: After release 24.3 use ignore-drop-queries-probability for both + # TODO: After release 24.3 use ignore_drop_queries_probability for both # stress test and upgrade check if not upgrade_check: - client_options.append("ignore-drop-queries-probability=0.5") + client_options.append("ignore_drop_queries_probability=0.5") if client_options: options.append(" --client-option " + " ".join(client_options)) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 83bf2f59131..edc7825896a 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -3171,7 +3171,7 @@ def parse_args(): ) # TODO: Remove upgrade-check option after release 24.3 and use - # ignore-drop-queries-probability option in stress.py as in stress tests + # ignore_drop_queries_probability option in stress.py as in stress tests group.add_argument( "--upgrade-check", action="store_true", diff --git a/tests/queries/0_stateless/03013_ignore_drop_queries_probability.reference b/tests/queries/0_stateless/03013_ignore_drop_queries_probability.reference new file mode 100644 index 00000000000..d81cc0710eb --- /dev/null +++ b/tests/queries/0_stateless/03013_ignore_drop_queries_probability.reference @@ -0,0 +1 @@ +42 diff --git a/tests/queries/0_stateless/03013_ignore_drop_queries_probability.sql b/tests/queries/0_stateless/03013_ignore_drop_queries_probability.sql new file mode 100644 index 00000000000..5c7b9998776 --- /dev/null +++ b/tests/queries/0_stateless/03013_ignore_drop_queries_probability.sql @@ -0,0 +1,18 @@ +create table test_memory (number UInt64) engine=Memory; +insert into test_memory select 42; +drop table test_memory settings ignore_drop_queries_probability=1; +select * from test_memory; +drop table test_memory; + +create table test_merge_tree (number UInt64) engine=MergeTree order by number; +insert into test_merge_tree select 42; +drop table test_merge_tree settings ignore_drop_queries_probability=1; +select * from test_merge_tree; +drop table test_merge_tree; + +create table test_join (number UInt64) engine=Join(ALL, LEFT, number); +insert into test_join select 42; +drop table test_join settings ignore_drop_queries_probability=1; +select * from test_join; +drop table test_join; + From 90c4b0aaf0e5ed0823b24d6c5ce48e726accf451 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:52:21 +0000 Subject: [PATCH 0193/1165] Add word to aspell-dictg --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 57a8e0d5840..edc3d993ed9 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1581,6 +1581,7 @@ gccMurmurHash gcem generateRandom generateRandomStructure +generateSeries generateULID generateUUIDv geoDistance From 613c31038ae76be09d35a151e6446024b07f127b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 18:40:03 +0100 Subject: [PATCH 0194/1165] Fix recompilation of regexps during backtracking in the parser --- src/Analyzer/MatcherNode.cpp | 23 +++++--- src/Analyzer/MatcherNode.h | 13 ++--- src/Analyzer/QueryTreeBuilder.cpp | 11 +--- .../JoinToSubqueryTransformVisitor.cpp | 12 +++- .../TranslateQualifiedNamesVisitor.cpp | 15 ++++- src/Parsers/ASTColumnsMatcher.cpp | 57 +++++-------------- src/Parsers/ASTColumnsMatcher.h | 18 ++---- src/Parsers/ExpressionElementParsers.cpp | 23 ++++---- 8 files changed, 73 insertions(+), 99 deletions(-) diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index ae2b654ab13..d0efb0646b3 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include @@ -14,9 +13,15 @@ #include #include + namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_COMPILE_REGEXP; +} + const char * toString(MatcherNodeType matcher_node_type) { switch (matcher_node_type) @@ -48,20 +53,20 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, ColumnTransformersNod { } -MatcherNode::MatcherNode(std::shared_ptr columns_matcher_, ColumnTransformersNodes column_transformers_) +MatcherNode::MatcherNode(const String & pattern_, ColumnTransformersNodes column_transformers_) : MatcherNode(MatcherNodeType::COLUMNS_REGEXP, {} /*qualified_identifier*/, {} /*columns_identifiers*/, - std::move(columns_matcher_), + std::move(pattern_), std::move(column_transformers_)) { } -MatcherNode::MatcherNode(Identifier qualified_identifier_, std::shared_ptr columns_matcher_, ColumnTransformersNodes column_transformers_) +MatcherNode::MatcherNode(Identifier qualified_identifier_, const String & pattern_, ColumnTransformersNodes column_transformers_) : MatcherNode(MatcherNodeType::COLUMNS_REGEXP, std::move(qualified_identifier_), {} /*columns_identifiers*/, - std::move(columns_matcher_), + std::move(pattern_), std::move(column_transformers_)) { } @@ -87,14 +92,18 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, Identifiers columns_i MatcherNode::MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - std::shared_ptr columns_matcher_, + const String & pattern_, ColumnTransformersNodes column_transformers_) : IQueryTreeNode(children_size) , matcher_type(matcher_type_) , qualified_identifier(qualified_identifier_) , columns_identifiers(columns_identifiers_) - , columns_matcher(columns_matcher_) { + columns_matcher = std::make_shared(pattern_, re2::RE2::Quiet); + if (!columns_matcher->ok()) + throw DB::Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern_, columns_matcher->error()); + auto column_transformers_list_node = std::make_shared(); auto & column_transformers_nodes = column_transformers_list_node->getNodes(); diff --git a/src/Analyzer/MatcherNode.h b/src/Analyzer/MatcherNode.h index 87ee5ee92bb..2217fde6eff 100644 --- a/src/Analyzer/MatcherNode.h +++ b/src/Analyzer/MatcherNode.h @@ -6,6 +6,7 @@ #include #include + namespace DB { @@ -58,10 +59,10 @@ public: explicit MatcherNode(Identifier qualified_identifier_, ColumnTransformersNodes column_transformers_ = {}); /// Variant unqualified COLUMNS('regexp') - explicit MatcherNode(std::shared_ptr columns_matcher_, ColumnTransformersNodes column_transformers_ = {}); + explicit MatcherNode(const String & pattern_, ColumnTransformersNodes column_transformers_ = {}); /// Variant qualified COLUMNS('regexp') - explicit MatcherNode(Identifier qualified_identifier_, std::shared_ptr columns_matcher_, ColumnTransformersNodes column_transformers_ = {}); + explicit MatcherNode(Identifier qualified_identifier_, const String & pattern_, ColumnTransformersNodes column_transformers_ = {}); /// Variant unqualified COLUMNS(column_name_1, ...) explicit MatcherNode(Identifiers columns_identifiers_, ColumnTransformersNodes column_transformers_ = {}); @@ -81,12 +82,6 @@ public: return matcher_type == MatcherNodeType::ASTERISK; } - /// Returns true if matcher is columns regexp or columns list matcher, false otherwise - bool isColumnsMatcher() const - { - return matcher_type == MatcherNodeType::COLUMNS_REGEXP || matcher_type == MatcherNodeType::COLUMNS_LIST; - } - /// Returns true if matcher is qualified, false otherwise bool isQualified() const { @@ -152,7 +147,7 @@ private: explicit MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - std::shared_ptr columns_matcher_, + const String & pattern_, ColumnTransformersNodes column_transformers_); MatcherNodeType matcher_type; diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 14559cca887..df80f46b3cd 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -2,11 +2,7 @@ #include -#include -#include -#include #include -#include #include #include #include @@ -33,13 +29,11 @@ #include #include #include -#include #include #include #include #include #include -#include #include #include #include @@ -50,7 +44,6 @@ #include #include -#include namespace DB @@ -661,7 +654,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co else if (const auto * columns_regexp_matcher = expression->as()) { auto column_transformers = buildColumnTransformers(columns_regexp_matcher->transformers, context); - result = std::make_shared(columns_regexp_matcher->getMatcher(), std::move(column_transformers)); + result = std::make_shared(columns_regexp_matcher->getPattern(), std::move(column_transformers)); } else if (const auto * columns_list_matcher = expression->as()) { @@ -681,7 +674,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co { auto & qualified_identifier = qualified_columns_regexp_matcher->qualifier->as(); auto column_transformers = buildColumnTransformers(qualified_columns_regexp_matcher->transformers, context); - result = std::make_shared(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getMatcher(), std::move(column_transformers)); + result = std::make_shared(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getPattern(), std::move(column_transformers)); } else if (const auto * qualified_columns_list_matcher = expression->as()) { diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 5c4ae528fc1..209f8e68b8f 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -15,11 +15,10 @@ #include #include #include -#include #include -#include #include #include +#include namespace DB { @@ -30,6 +29,7 @@ namespace ErrorCodes extern const int AMBIGUOUS_COLUMN_NAME; extern const int NOT_IMPLEMENTED; extern const int UNKNOWN_IDENTIFIER; + extern const int CANNOT_COMPILE_REGEXP; } namespace @@ -204,11 +204,17 @@ private: { has_asterisks = true; + String pattern = columns_regexp_matcher->getPattern(); + auto regexp = std::make_shared(pattern, re2::RE2::Quiet); + if (!regexp->ok()) + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + for (auto & table_name : data.tables_order) data.addTableColumns( table_name, columns, - [&](const String & column_name) { return columns_regexp_matcher->isColumnMatching(column_name); }); + [&](const String & column_name) { return re2::RE2::PartialMatch(column_name, *regexp); }); if (columns_regexp_matcher->transformers) { diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 3de7e217e53..e3690d0fcc0 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -1,4 +1,3 @@ -#include #include #include @@ -6,7 +5,6 @@ #include #include -#include #include #include @@ -21,6 +19,7 @@ #include #include #include +#include namespace DB @@ -31,7 +30,10 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int UNSUPPORTED_JOIN_KEYS; extern const int LOGICAL_ERROR; + extern const int CANNOT_COMPILE_REGEXP; } + + bool TranslateQualifiedNamesMatcher::Data::matchColumnName(std::string_view name, const String & column_name, DataTypePtr column_type) { if (name.size() < column_name.size()) @@ -275,12 +277,19 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt } else if (const auto * asterisk_regexp_pattern = child->as()) { + String pattern = asterisk_regexp_pattern->getPattern(); + auto regexp = std::make_shared(pattern, re2::RE2::Quiet); + if (!regexp->ok()) + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + bool first_table = true; for (const auto & table : tables_with_columns) { for (const auto & column : table.columns) { - if (asterisk_regexp_pattern->isColumnMatching(column.name) && (first_table || !data.join_using_columns.contains(column.name))) + if (re2::RE2::PartialMatch(column.name, *regexp) + && (first_table || !data.join_using_columns.contains(column.name))) { addIdentifier(columns, table.table, column.name); } diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp index c2854e2235c..dac3661d482 100644 --- a/src/Parsers/ASTColumnsMatcher.cpp +++ b/src/Parsers/ASTColumnsMatcher.cpp @@ -34,14 +34,14 @@ void ASTColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) const writeCString(".", ostr); } writeCString("COLUMNS(", ostr); - writeQuotedString(original_pattern, ostr); + writeQuotedString(pattern, ostr); writeChar(')', ostr); } void ASTColumnsRegexpMatcher::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const { - hash_state.update(original_pattern.size()); - hash_state.update(original_pattern); + hash_state.update(pattern.size()); + hash_state.update(pattern); IAST::updateTreeHashImpl(hash_state, ignore_aliases); } @@ -56,7 +56,7 @@ void ASTColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, Format } settings.ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "("; - settings.ostr << quoteString(original_pattern); + settings.ostr << quoteString(pattern); settings.ostr << ")"; if (transformers) @@ -65,28 +65,14 @@ void ASTColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, Format } } -void ASTColumnsRegexpMatcher::setPattern(String pattern) +void ASTColumnsRegexpMatcher::setPattern(String pattern_) { - original_pattern = std::move(pattern); - column_matcher = std::make_shared(original_pattern, RE2::Quiet); - if (!column_matcher->ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", original_pattern, column_matcher->error()); + pattern = std::move(pattern_); } const String & ASTColumnsRegexpMatcher::getPattern() const { - return original_pattern; -} - -const std::shared_ptr & ASTColumnsRegexpMatcher::getMatcher() const -{ - return column_matcher; -} - -bool ASTColumnsRegexpMatcher::isColumnMatching(const String & column_name) const -{ - return RE2::PartialMatch(column_name, *column_matcher); + return pattern; } ASTPtr ASTColumnsListMatcher::clone() const @@ -166,37 +152,24 @@ void ASTQualifiedColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) cons { qualifier->appendColumnName(ostr); writeCString(".COLUMNS(", ostr); - writeQuotedString(original_pattern, ostr); + writeQuotedString(pattern, ostr); writeChar(')', ostr); } -void ASTQualifiedColumnsRegexpMatcher::setPattern(String pattern, bool set_matcher) +void ASTQualifiedColumnsRegexpMatcher::setPattern(String pattern_) { - original_pattern = std::move(pattern); - - if (!set_matcher) - return; - - column_matcher = std::make_shared(original_pattern, RE2::Quiet); - if (!column_matcher->ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", original_pattern, column_matcher->error()); + pattern = std::move(pattern_); } -void ASTQualifiedColumnsRegexpMatcher::setMatcher(std::shared_ptr matcher) +const String & ASTQualifiedColumnsRegexpMatcher::getPattern() const { - column_matcher = std::move(matcher); -} - -const std::shared_ptr & ASTQualifiedColumnsRegexpMatcher::getMatcher() const -{ - return column_matcher; + return pattern; } void ASTQualifiedColumnsRegexpMatcher::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const { - hash_state.update(original_pattern.size()); - hash_state.update(original_pattern); + hash_state.update(pattern.size()); + hash_state.update(pattern); IAST::updateTreeHashImpl(hash_state, ignore_aliases); } @@ -207,7 +180,7 @@ void ASTQualifiedColumnsRegexpMatcher::formatImpl(const FormatSettings & setting qualifier->formatImpl(settings, state, frame); settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "("; - settings.ostr << quoteString(original_pattern); + settings.ostr << quoteString(pattern); settings.ostr << ")"; if (transformers) diff --git a/src/Parsers/ASTColumnsMatcher.h b/src/Parsers/ASTColumnsMatcher.h index 6fc5581a4eb..e5191b74b72 100644 --- a/src/Parsers/ASTColumnsMatcher.h +++ b/src/Parsers/ASTColumnsMatcher.h @@ -2,11 +2,6 @@ #include -namespace re2 -{ -class RE2; -} - namespace DB { @@ -25,8 +20,6 @@ public: void appendColumnName(WriteBuffer & ostr) const override; void setPattern(String pattern); const String & getPattern() const; - const std::shared_ptr & getMatcher() const; - bool isColumnMatching(const String & column_name) const; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; ASTPtr expression; @@ -35,8 +28,7 @@ protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: - std::shared_ptr column_matcher; - String original_pattern; + String pattern; }; /// Same as the above but use a list of column names to do matching. @@ -62,9 +54,8 @@ public: ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; - const std::shared_ptr & getMatcher() const; - void setPattern(String pattern, bool set_matcher = true); - void setMatcher(std::shared_ptr matcher); + void setPattern(String pattern_); + const String & getPattern() const; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; ASTPtr qualifier; @@ -73,8 +64,7 @@ protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: - std::shared_ptr column_matcher; - String original_pattern; + String pattern; }; /// Same as ASTColumnsListMatcher. Qualified identifier is first child. diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 1135fe88c16..67f4a306292 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include @@ -44,6 +43,7 @@ #include + namespace DB { @@ -1628,7 +1628,7 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e is_strict = true; ASTs identifiers; - ASTPtr regex_node; + ASTPtr regexp_node; ParserStringLiteral regex; auto parse_id = [&identifiers, &pos, &expected] { @@ -1644,7 +1644,7 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e { // support one or more parameter ++pos; - if (!ParserList::parseUtil(pos, expected, parse_id, false) && !regex.parse(pos, regex_node, expected)) + if (!ParserList::parseUtil(pos, expected, parse_id, false) && !regex.parse(pos, regexp_node, expected)) return false; if (pos->type != TokenType::ClosingRoundBracket) @@ -1654,13 +1654,13 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e else { // only one parameter - if (!parse_id() && !regex.parse(pos, regex_node, expected)) + if (!parse_id() && !regex.parse(pos, regexp_node, expected)) return false; } auto res = std::make_shared(); - if (regex_node) - res->setPattern(regex_node->as().value.get()); + if (regexp_node) + res->setPattern(regexp_node->as().value.get()); else res->children = std::move(identifiers); res->is_strict = is_strict; @@ -1794,11 +1794,11 @@ static bool parseColumnsMatcherBody(IParser::Pos & pos, ASTPtr & node, Expected ++pos; ParserList columns_p(std::make_unique(false, true), std::make_unique(TokenType::Comma), false); - ParserStringLiteral regex; + ParserStringLiteral regexp; ASTPtr column_list; - ASTPtr regex_node; - if (!columns_p.parse(pos, column_list, expected) && !regex.parse(pos, regex_node, expected)) + ASTPtr regexp_node; + if (!columns_p.parse(pos, column_list, expected) && !regexp.parse(pos, regexp_node, expected)) return false; if (pos->type != TokenType::ClosingRoundBracket) @@ -1832,7 +1832,7 @@ static bool parseColumnsMatcherBody(IParser::Pos & pos, ASTPtr & node, Expected else { auto regexp_matcher = std::make_shared(); - regexp_matcher->setPattern(regex_node->as().value.get()); + regexp_matcher->setPattern(regexp_node->as().value.get()); if (!transformers->children.empty()) { @@ -1895,8 +1895,7 @@ bool ParserQualifiedColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected else if (auto * column_regexp_matcher = node->as()) { auto result = std::make_shared(); - result->setPattern(column_regexp_matcher->getPattern(), false); - result->setMatcher(column_regexp_matcher->getMatcher()); + result->setPattern(column_regexp_matcher->getPattern()); result->qualifier = std::move(identifier_node); result->children.push_back(result->qualifier); From 4a58705f7bcfcb2b4a66bf8def54297a6dc64ef2 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 15 Mar 2024 18:08:41 +0000 Subject: [PATCH 0195/1165] impl trigger rebuild fix move fix black trigger rebuild trigger rebuild fix mypy fix fix --- tests/ci/ci.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 29906e6571f..795c0d7b7c6 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1613,6 +1613,42 @@ def _upload_build_profile_data( logging.error("Failed to insert binary_size_file for the build, continue") +def _add_build_to_version_history( + pr_info: PRInfo, + job_report: JobReport, + git_ref: str, + version: str, + ch_helper: ClickHouseHelper, +) -> None: + ci_logs_credentials = CiLogsCredentials(Path("/dev/null")) + if not ci_logs_credentials.host: + return + + # with some probability we will not silently break this logic + assert pr_info.sha and pr_info.commit_html_url and version and git_ref + + data = { + "check_start_time": job_report.start_time, + "pull_request_number": pr_info.number, + "pull_request_url": pr_info.pr_html_url, + "commit_sha": pr_info.sha, + "commit_url": pr_info.commit_html_url, + "version": version, + "git_ref": git_ref, + } + + json_str = json.dumps(data) + + print(f"::notice ::Log Adding record to versions history: {json_str}") + + try: + ch_helper.insert_json_into( + db="default", table="version_history", json_str=json_str + ) + except InsertException: + logging.error("Failed to insert profile data for the build, continue") + + def _run_test(job_name: str, run_command: str) -> int: assert ( run_command or CI_CONFIG.get_job_config(job_name).run_command @@ -1986,6 +2022,11 @@ def main() -> int: ch_helper.insert_events_into( db="default", table="checks", events=prepared_events ) + + if args.job_name == "DockerServerImageRelease" and indata is not None: + _add_build_to_version_history( + pr_info, job_report, indata["git_ref"], indata["version"], ch_helper + ) else: # no job report print(f"No job report for {[args.job_name]} - do nothing") From ac352b96f038ab3f393ac330097f8dc03db31001 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 18 Mar 2024 15:26:33 +0000 Subject: [PATCH 0196/1165] remove copy-paste --- tests/ci/ci.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 795c0d7b7c6..c5fa67a66cd 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1620,10 +1620,6 @@ def _add_build_to_version_history( version: str, ch_helper: ClickHouseHelper, ) -> None: - ci_logs_credentials = CiLogsCredentials(Path("/dev/null")) - if not ci_logs_credentials.host: - return - # with some probability we will not silently break this logic assert pr_info.sha and pr_info.commit_html_url and version and git_ref From 0b2c2741320ac542ccb3e744e8849ca4767e49ae Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 18 Mar 2024 17:45:23 +0000 Subject: [PATCH 0197/1165] write docker tag --- tests/ci/ci.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index c5fa67a66cd..f2e0828082c 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1618,6 +1618,7 @@ def _add_build_to_version_history( job_report: JobReport, git_ref: str, version: str, + docker_tag: str, ch_helper: ClickHouseHelper, ) -> None: # with some probability we will not silently break this logic @@ -1630,6 +1631,7 @@ def _add_build_to_version_history( "commit_sha": pr_info.sha, "commit_url": pr_info.commit_html_url, "version": version, + "docker_tag": docker_tag, "git_ref": git_ref, } @@ -2019,9 +2021,14 @@ def main() -> int: db="default", table="checks", events=prepared_events ) - if args.job_name == "DockerServerImageRelease" and indata is not None: + if "DockerServerImage" in args.job_name and indata is not None: _add_build_to_version_history( - pr_info, job_report, indata["git_ref"], indata["version"], ch_helper + pr_info, + job_report, + indata["git_ref"], + indata["version"], + indata["build"], + ch_helper, ) else: # no job report From 05ee8369b298cb7091ac621bee4098f801353f3f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 18:54:13 +0100 Subject: [PATCH 0198/1165] Add a test --- .../03013_parser_regexp_recompilation.reference | 1 + .../0_stateless/03013_parser_regexp_recompilation.sh | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03013_parser_regexp_recompilation.reference create mode 100755 tests/queries/0_stateless/03013_parser_regexp_recompilation.sh diff --git a/tests/queries/0_stateless/03013_parser_regexp_recompilation.reference b/tests/queries/0_stateless/03013_parser_regexp_recompilation.reference new file mode 100644 index 00000000000..8e0782c44f2 --- /dev/null +++ b/tests/queries/0_stateless/03013_parser_regexp_recompilation.reference @@ -0,0 +1 @@ +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03013_parser_regexp_recompilation.sh b/tests/queries/0_stateless/03013_parser_regexp_recompilation.sh new file mode 100755 index 00000000000..ea7a72ce73d --- /dev/null +++ b/tests/queries/0_stateless/03013_parser_regexp_recompilation.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# This query is incorrect, but its parsing should not be slow (tens of seconds): + +${CLICKHOUSE_LOCAL} < "$CURDIR"/03013_parser_regexp_recompilation.txt 2>&1 | grep --text -o -F 'SYNTAX_ERROR' From 597b72dc5221fdcd2d4e87f24489d0d164a90527 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 18:54:28 +0100 Subject: [PATCH 0199/1165] Add a test --- .../03013_parser_regexp_recompilation.txt | Bin 0 -> 55948 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/queries/0_stateless/03013_parser_regexp_recompilation.txt diff --git a/tests/queries/0_stateless/03013_parser_regexp_recompilation.txt b/tests/queries/0_stateless/03013_parser_regexp_recompilation.txt new file mode 100644 index 0000000000000000000000000000000000000000..a861059c3a1322eae212bca0f4b5fc2e1be7de1e GIT binary patch literal 55948 zcmeHQUymEN5tmcMD1^38MNyy#ut5Re-R5lEAEQYc7{SF>f& z=)70Fm`;8eT#kp6qIde;@oBI4)6vPv_Ty(K&!3*2Z*Sf9yX-2vzv|w;yVb64_W@Ml zbP%?m zABexEgk}%Akip624<;s*YKZacMd!E4FYaVM;@YvUzVjX4_?VUPq5V9^DnOwpI)+RG zua*EL$5?Nt>y%O;3zr^JdBGB>IG_Pw*~Mx~sO7wlEGup9Ezkv*?g!pg)mgOKpq`vO zTRV<9tF?|2TrM*o<22_>-x~xPsLQC+Zc*r7bzu>wny~){+f-WaVUb3KhPz+O!Us>p zGaHA2jCg(OSHF)deYK7W=K|ELS?!&9<-6GwpXw}#ikSctLv-;q4yi6<+%0*twzlTvP|SC zYWkj&V~SJGjIg|$hlsN-e~gwO%N@%oiPSjE&z?OkUXLcD*(dYS*>Li5dbzz*oE$$r z?iYt&b&pTaduRRP__Y75m=6Z8uHU{L>=oy|liuTgQ5N5-*28e?-J0e%gZa&y(N*!N zn9r_br^)Xff}c9dXwhi57)^%XUyKGX$CKg3cz$2&8mq4dZ>Cpc+R)F>$5WFFbFC9s&g^CVk$4DuLu9Q9?iyLg!lLM?(gjvdwYN6|84CSThIHCw|0u7^J2bR z%!|*8eMY8F7^vIxo#HbnVJ_&M{rTwRudeIrXmV41fav_9&lG*X|5dm9SbVx*A1+RM zP*iMBis#3Z`QgECF)WT|v*FG5w-?jl{0|Rzb{I0R$CJ^cr=!d1>_$1??|WC6{LK|6 zZ=1wPO_)SPSxsxr(|&(?YD8=Z~szPz!Cy@7L^M@G^mbI*D09~^x7`47a2+3&IMGMM6cM_aENV$+U zO#(MkYj_qx4{27QRC-D<3G83kN``)A{Xl2%Cw`&ThAClANm`AX4P=?fQPkp&2d(xd z*A4{ra1gx_)8Zq-nb4=W{L`=m0q$5vi6wEEtHUyz@s;S)z-)_rQ@G%-0QpGP!Vdnq zKGMohy!l9${Lz4XX3u@Dv|WG%G7qV)l2HCk1gC+Qk9(ufM0K0?pWbI8Jd3TJe91?$ z>{IF=JgW1NEc)*WnxDuzbf3O*mgXZ_WPOutwanF~+q~D#N3x9I$w#vCzmG9O;(c$c zB$W5sYp~a@bMsGcuT9@Y=ObC{Ehl@!S+YO$H7DHqU#t3SN@zx2m%?p88iF(8rETz4 zt>`MyS+r?EJ^2@HT04$;thJ63+|B|>Uj8Uy0@C@8KJ$?*l>{8{8DB3)J#5mCa_)I;9x`W&p74 zz5+kR#}+2JB{c1&_onTnuI}p;5o>TrO>1S`I1SOJN#I86Th&=)3|HLZpjEz$IGr5r z7U%*y5R`WsfKjU!R(6tO`ZZUq8wYphyg^(vNoiD5f;vzqqgdpkrYXMe%(g_Xk*7Kx z*h!Msl6C6WoD{f`dXoE4=rzr=eP|EK-IB;TU?8=TtY%qV8x-^fMqOJ{7gB@uqDlTI z0t7nFB0fk-efoG{opAkuxuaD}gEn#^WrRt%SuBPmQX#42oW0gg z+?KPp6jUqarEh7G;?eI-R*2St0lj0#NV!D>q97}W&@gAoRt|!OASEK}JYXD-qI_Hn z2rs<@fnGFl(SpP)7OO0f3>OH-5fP9Y8-|qGtyq3n4Pgr{S&)-72>^|yh%Mg7v^O7b zTEXQsn*%peaSm_`VJEagDmphs3tGLC0;eGXw`$c=*Q)xM47Fl$&FakMM|H zL*THPPL)T1(W~@36!Zm#F73Swslj@oAV0#JKq5X!NqzcwV4ZON&OM~3P#Tnc5fplH z*cDx72FeJNaI@ITCxRmat$|y!M|d)H!b*Ug6NiXd=NH5zEU5_RO~4zUBD6F@0cvfM z=v(`jc8msa>zrVD3V8$XUn^zfs-{OLdgzd~jfmuE$L`FXaI!*)s56df8$@1FUYRq2 z6F5>RG5lbD$C;x11`Ks<10HptPDZiFMVGlB5~IjZWa!QGOAKmuQS`nqFkEdMWg#_K zFPi2jG800?2Pvsf9}lb(uAk((AS(?@o(zRv9Ck&QnSnCGB-|{v@`>PxKx^RE=ZOr) zuKxf5?7vF8n3cd<#s~nV{j!X|rkNftZE0_+Lu%>D%V%^mwZS3g->qrwIOed{I=ENW zBZMBYpxnM>mxSvl~st*h9a`s8D_TvfRc)E-jZ97f77R%%}1i8p1_TSA$ews zec^}efHn-vH*p2119dWrMJ~Gh)g{09K}V)fD5%**(fhu@aJ6xih16iZXqsRAm=Gd9 zNJ)MAcwn7y{ek?aRZD}CCqtnZhh5QSW}u8P2{((ad?Gj^&>Fb)dGW(X?Sw*`CV?A? z-Kx$a;@XPEK{r_;S_cMnj~yfBCItZ2s^u6OqAgoF2pWQvh^+H~aX5q>2i6JK?@D^UGHFoqWGM9Fuq&Eh7Wq8!fxh=B z*Q5;1yiN0YnV_sEbMOPNEU5_Rm5UV9a28n_p#YUNSs^w%XT@&doN!i%I$clOAo7ax z%ItS1aHNo8X!DyLn_|2wVic1z0YfJF?j>ge2rzq&IsZ==Tw?WQDtaZurO8AOvQqsA z^E Date: Mon, 18 Mar 2024 18:15:54 +0000 Subject: [PATCH 0200/1165] add logging --- src/Common/AsyncLoader.cpp | 35 ++++++++++++++++++++++--- src/Common/AsyncLoader.h | 4 ++- src/Common/tests/gtest_async_loader.cpp | 2 +- src/Interpreters/Context.cpp | 3 ++- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index 4a39454ccbb..e4896ce32ab 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -196,9 +197,10 @@ void LoadTask::remove() } } -AsyncLoader::AsyncLoader(std::vector pool_initializers, bool log_failures_, bool log_progress_) +AsyncLoader::AsyncLoader(std::vector pool_initializers, bool log_failures_, bool log_progress_, bool log_events_) : log_failures(log_failures_) , log_progress(log_progress_) + , log_events(log_events_) , log(getLogger("AsyncLoader")) { pools.reserve(pool_initializers.size()); @@ -327,6 +329,8 @@ void AsyncLoader::schedule(const LoadJobSet & jobs_to_schedule) ALLOW_ALLOCATIONS_IN_SCOPE; scheduled_jobs.try_emplace(job); job->scheduled(++last_job_id); + if (log_events) + LOG_DEBUG(log, "Schedule load job '{}' into {}", job->name, getPoolName(job->pool())); }); } @@ -582,6 +586,9 @@ void AsyncLoader::finish(const LoadJobPtr & job, LoadStatus status, std::excepti else if (status == LoadStatus::CANCELED) job->canceled(reason); + if (log_events) + LOG_DEBUG(log, "Finish load job '{}' with status {}", job->name, magic_enum::enum_name(status)); + Info & info = scheduled_jobs[job]; if (info.isReady()) { @@ -661,6 +668,9 @@ void AsyncLoader::prioritize(const LoadJobPtr & job, size_t new_pool_id, std::un job->pool_id.store(new_pool_id); + if (log_events) + LOG_DEBUG(log, "Prioritize load job '{}': {} -> {}", job->name, old_pool.name, new_pool.name); + // Recurse into dependencies for (const auto & dep : job->dependencies) prioritize(dep, new_pool_id, lock); @@ -765,6 +775,9 @@ void AsyncLoader::wait(std::unique_lock & job_lock, const LoadJobPtr if (job->load_status != LoadStatus::PENDING) // Shortcut just to avoid incrementing ProfileEvents return; + if (log_events) + LOG_DEBUG(log, "Wait load job '{}' in {}", job->name, getPoolName(job->executionPool())); + Stopwatch watch; job->waiters++; job->finished.wait(job_lock, [&] { return job->load_status != LoadStatus::PENDING; }); @@ -789,6 +802,13 @@ bool AsyncLoader::canWorkerLive(Pool & pool, std::unique_lock &) && (!current_priority || *current_priority >= pool.priority); } +void AsyncLoader::setCurrentPriority(std::unique_lock &, std::optional priority) +{ + if (log_event && current_priority != priority) + LOG_DEBUG(log, "Change current priority: {} -> {}", current_priority, priority); + current_priority = priority; +} + void AsyncLoader::updateCurrentPriorityAndSpawn(std::unique_lock & lock) { // Find current priority. @@ -799,7 +819,7 @@ void AsyncLoader::updateCurrentPriorityAndSpawn(std::unique_lock & l if (pool.isActive() && (!priority || *priority > pool.priority)) priority = pool.priority; } - current_priority = priority; + setCurrentPriority(lock, priority); // Spawn workers in all pools with current priority for (Pool & pool : pools) @@ -809,10 +829,12 @@ void AsyncLoader::updateCurrentPriorityAndSpawn(std::unique_lock & l } } -void AsyncLoader::spawn(Pool & pool, std::unique_lock &) +void AsyncLoader::spawn(Pool & pool, std::unique_lock & lock) { + setCurrentPriority(lock, pool.priority); // canSpawnWorker() ensures this would not decrease current_priority pool.workers++; - current_priority = pool.priority; // canSpawnWorker() ensures this would not decrease current_priority + if (log_events) + LOG_DEBUG(log, "Spawn loader worker #{} in {}", pool.workers, pool.name); NOEXCEPT_SCOPE({ ALLOW_ALLOCATIONS_IN_SCOPE; pool.thread_pool->scheduleOrThrowOnError([this, &pool] { worker(pool); }); @@ -842,6 +864,8 @@ void AsyncLoader::worker(Pool & pool) if (!canWorkerLive(pool, lock)) { + if (log_events) + LOG_DEBUG(log, "Stop worked in {}", pool.name); if (--pool.workers == 0) updateCurrentPriorityAndSpawn(lock); // It will spawn lower priority workers if needed return; @@ -852,6 +876,9 @@ void AsyncLoader::worker(Pool & pool) job = it->second; pool.ready_queue.erase(it); scheduled_jobs.find(job)->second.ready_seqno = 0; // This job is no longer in the ready queue + + if (log_events) + LOG_DEBUG(log, "Execute load job '{}' in {}", job->name, pool.name); } ALLOW_ALLOCATIONS_IN_SCOPE; diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h index b1b336d24dc..a5c7049ee0e 100644 --- a/src/Common/AsyncLoader.h +++ b/src/Common/AsyncLoader.h @@ -327,7 +327,7 @@ private: }; public: - AsyncLoader(std::vector pool_initializers, bool log_failures_, bool log_progress_); + AsyncLoader(std::vector pool_initializers, bool log_failures_, bool log_progress_, bool log_events_); // WARNING: all tasks instances should be destructed before associated AsyncLoader. ~AsyncLoader(); @@ -407,6 +407,7 @@ private: void wait(std::unique_lock & job_lock, const LoadJobPtr & job); bool canSpawnWorker(Pool & pool, std::unique_lock & lock); bool canWorkerLive(Pool & pool, std::unique_lock & lock); + void setCurrentPriority(std::unique_lock & lock, std::optional priority); void updateCurrentPriorityAndSpawn(std::unique_lock & lock); void spawn(Pool & pool, std::unique_lock & lock); void worker(Pool & pool); @@ -415,6 +416,7 @@ private: // Logging const bool log_failures; // Worker should log all exceptions caught from job functions. const bool log_progress; // Periodically log total progress + const bool log_events; // Log all important events: job start/end, waits, prioritizations LoggerPtr log; mutable std::mutex mutex; // Guards all the fields below. diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp index fc2537abcfc..b7b14bbedd4 100644 --- a/src/Common/tests/gtest_async_loader.cpp +++ b/src/Common/tests/gtest_async_loader.cpp @@ -50,7 +50,7 @@ struct AsyncLoaderTest pcg64 rng{randomSeed()}; explicit AsyncLoaderTest(std::vector initializers) - : loader(getPoolInitializers(initializers), /* log_failures = */ false, /* log_progress = */ false) + : loader(getPoolInitializers(initializers), /* log_failures = */ false, /* log_progress = */ false, /* log_events = */ false) { loader.stop(); // All tests call `start()` manually to better control ordering } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6a0657a842c..48c0618f5a9 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2461,7 +2461,8 @@ AsyncLoader & Context::getAsyncLoader() const } }, /* log_failures = */ true, - /* log_progress = */ true); + /* log_progress = */ true, + /* log_events = */ true); }); return *shared->async_loader; From 46debe89c22e8f423c66c380caf3e90642e81a77 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 18 Mar 2024 18:17:55 +0000 Subject: [PATCH 0201/1165] fix --- src/Databases/DatabaseReplicated.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 0c2cf1bb011..ffaefe08b5f 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1329,7 +1329,6 @@ void DatabaseReplicated::stopReplication() void DatabaseReplicated::shutdown() { - stopLoading(); stopReplication(); ddl_worker_initialized = false; ddl_worker = nullptr; From 4f30ac72a1ef01f7c783c192639933e32d2f0397 Mon Sep 17 00:00:00 2001 From: William Schoeffel Date: Mon, 18 Mar 2024 15:18:47 -0300 Subject: [PATCH 0202/1165] fix typo in variable name legacy --- src/Storages/MergeTree/MergeTreePartInfo.cpp | 10 +++++----- src/Storages/MergeTree/MergeTreePartInfo.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreePartInfo.cpp b/src/Storages/MergeTree/MergeTreePartInfo.cpp index 8c1b630e85e..f3a974a22af 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -131,7 +131,7 @@ std::optional MergeTreePartInfo::tryParsePartName( /// "Part 20170601_20170630_0_2_999999999 intersects 201706_0_1_4294967295". /// So we replace unexpected max level to make contains(...) method and comparison operators work /// correctly with such virtual parts. On part name serialization we will use legacy max level to keep the name unchanged. - part_info.use_leagcy_max_level = true; + part_info.use_legacy_max_level = true; level = MAX_LEVEL; } @@ -205,7 +205,7 @@ String MergeTreePartInfo::getPartNameV1() const writeChar('_', wb); writeIntText(max_block, wb); writeChar('_', wb); - if (use_leagcy_max_level) + if (use_legacy_max_level) { assert(level == MAX_LEVEL); writeIntText(LEGACY_MAX_LEVEL, wb); @@ -244,7 +244,7 @@ String MergeTreePartInfo::getPartNameV0(DayNum left_date, DayNum right_date) con writeChar('_', wb); writeIntText(max_block, wb); writeChar('_', wb); - if (use_leagcy_max_level) + if (use_legacy_max_level) { assert(level == MAX_LEVEL); writeIntText(LEGACY_MAX_LEVEL, wb); @@ -274,7 +274,7 @@ void MergeTreePartInfo::serialize(WriteBuffer & out) const writeIntBinary(max_block, out); writeIntBinary(level, out); writeIntBinary(mutation, out); - writeBoolText(use_leagcy_max_level, out); + writeBoolText(use_legacy_max_level, out); } @@ -297,7 +297,7 @@ void MergeTreePartInfo::deserialize(ReadBuffer & in) readIntBinary(max_block, in); readIntBinary(level, in); readIntBinary(mutation, in); - readBoolText(use_leagcy_max_level, in); + readBoolText(use_legacy_max_level, in); } bool MergeTreePartInfo::areAllBlockNumbersCovered(const MergeTreePartInfo & blocks_range, std::vector candidates) diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index 5fbb5d70bf3..9bb79e21144 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -26,7 +26,7 @@ struct MergeTreePartInfo UInt32 level = 0; Int64 mutation = 0; /// If the part has been mutated or contains mutated parts, is equal to mutation version number. - bool use_leagcy_max_level = false; /// For compatibility. TODO remove it + bool use_legacy_max_level = false; /// For compatibility. TODO remove it MergeTreePartInfo() = default; From e2c692e194e8f02a623a21568d1b50980754b019 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 18 Mar 2024 18:24:36 +0000 Subject: [PATCH 0203/1165] fix build --- src/Common/AsyncLoader.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index e4896ce32ab..02dada7b8f9 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -804,8 +804,10 @@ bool AsyncLoader::canWorkerLive(Pool & pool, std::unique_lock &) void AsyncLoader::setCurrentPriority(std::unique_lock &, std::optional priority) { - if (log_event && current_priority != priority) - LOG_DEBUG(log, "Change current priority: {} -> {}", current_priority, priority); + if (log_events && current_priority != priority) + LOG_DEBUG(log, "Change current priority: {} -> {}", + current_priority ? std::to_string(*current_priority) : "none", + priority ? std::to_string(*priority) : "none"); current_priority = priority; } From 72932424eb00da56c0e6d2655d1ceb19fddcf86e Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 18 Mar 2024 19:36:45 +0000 Subject: [PATCH 0204/1165] fix issue with allocations --- src/Common/AsyncLoader.cpp | 43 ++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index 02dada7b8f9..7d420b3beef 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -587,7 +587,12 @@ void AsyncLoader::finish(const LoadJobPtr & job, LoadStatus status, std::excepti job->canceled(reason); if (log_events) - LOG_DEBUG(log, "Finish load job '{}' with status {}", job->name, magic_enum::enum_name(status)); + { + NOEXCEPT_SCOPE({ + ALLOW_ALLOCATIONS_IN_SCOPE; + LOG_DEBUG(log, "Finish load job '{}' with status {}", job->name, magic_enum::enum_name(status)); + }); + } Info & info = scheduled_jobs[job]; if (info.isReady()) @@ -669,7 +674,12 @@ void AsyncLoader::prioritize(const LoadJobPtr & job, size_t new_pool_id, std::un job->pool_id.store(new_pool_id); if (log_events) - LOG_DEBUG(log, "Prioritize load job '{}': {} -> {}", job->name, old_pool.name, new_pool.name); + { + NOEXCEPT_SCOPE({ + ALLOW_ALLOCATIONS_IN_SCOPE; + LOG_DEBUG(log, "Prioritize load job '{}': {} -> {}", job->name, old_pool.name, new_pool.name); + }); + } // Recurse into dependencies for (const auto & dep : job->dependencies) @@ -805,9 +815,14 @@ bool AsyncLoader::canWorkerLive(Pool & pool, std::unique_lock &) void AsyncLoader::setCurrentPriority(std::unique_lock &, std::optional priority) { if (log_events && current_priority != priority) - LOG_DEBUG(log, "Change current priority: {} -> {}", - current_priority ? std::to_string(*current_priority) : "none", - priority ? std::to_string(*priority) : "none"); + { + NOEXCEPT_SCOPE({ + ALLOW_ALLOCATIONS_IN_SCOPE; + LOG_DEBUG(log, "Change current priority: {} -> {}", + current_priority ? std::to_string(*current_priority) : "none", + priority ? std::to_string(*priority) : "none"); + }); + } current_priority = priority; } @@ -835,10 +850,10 @@ void AsyncLoader::spawn(Pool & pool, std::unique_lock & lock) { setCurrentPriority(lock, pool.priority); // canSpawnWorker() ensures this would not decrease current_priority pool.workers++; - if (log_events) - LOG_DEBUG(log, "Spawn loader worker #{} in {}", pool.workers, pool.name); NOEXCEPT_SCOPE({ ALLOW_ALLOCATIONS_IN_SCOPE; + if (log_events) + LOG_DEBUG(log, "Spawn loader worker #{} in {}", pool.workers, pool.name); pool.thread_pool->scheduleOrThrowOnError([this, &pool] { worker(pool); }); }); } @@ -867,7 +882,12 @@ void AsyncLoader::worker(Pool & pool) if (!canWorkerLive(pool, lock)) { if (log_events) - LOG_DEBUG(log, "Stop worked in {}", pool.name); + { + NOEXCEPT_SCOPE({ + ALLOW_ALLOCATIONS_IN_SCOPE; + LOG_DEBUG(log, "Stop worked in {}", pool.name); + }); + } if (--pool.workers == 0) updateCurrentPriorityAndSpawn(lock); // It will spawn lower priority workers if needed return; @@ -880,7 +900,12 @@ void AsyncLoader::worker(Pool & pool) scheduled_jobs.find(job)->second.ready_seqno = 0; // This job is no longer in the ready queue if (log_events) - LOG_DEBUG(log, "Execute load job '{}' in {}", job->name, pool.name); + { + NOEXCEPT_SCOPE({ + ALLOW_ALLOCATIONS_IN_SCOPE; + LOG_DEBUG(log, "Execute load job '{}' in {}", job->name, pool.name); + }); + } } ALLOW_ALLOCATIONS_IN_SCOPE; From 083d0776edb3ba6f6253c452a6b06ff0908ea0f9 Mon Sep 17 00:00:00 2001 From: Alexander Sapin Date: Mon, 18 Mar 2024 20:47:31 +0100 Subject: [PATCH 0205/1165] Wait for done mutation properly --- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index c41403e312b..c89be57dc07 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -615,8 +615,8 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( else if (mutation_pointer_value >= mutation_id) /// Maybe we already processed more fresh mutation break; /// (numbers like 0000000000 and 0000000001) - /// Replica can become inactive, so wait with timeout and recheck it - if (wait_event->tryWait(1000)) + /// Replica can become inactive, so wait with timeout, if nothing happened -> recheck it + if (!wait_event->tryWait(1000)) continue; /// Here we check mutation for errors on local replica. If they happen on this replica From 6aa88ed5e958899def7dd8d35ae686a4030588bf Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 18 Mar 2024 19:49:05 +0000 Subject: [PATCH 0206/1165] typo --- src/Common/AsyncLoader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index 7d420b3beef..52cfbba8e0c 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -786,7 +786,7 @@ void AsyncLoader::wait(std::unique_lock & job_lock, const LoadJobPtr return; if (log_events) - LOG_DEBUG(log, "Wait load job '{}' in {}", job->name, getPoolName(job->executionPool())); + LOG_DEBUG(log, "Wait load job '{}' in {}", job->name, getPoolName(job->pool_id)); Stopwatch watch; job->waiters++; @@ -885,7 +885,7 @@ void AsyncLoader::worker(Pool & pool) { NOEXCEPT_SCOPE({ ALLOW_ALLOCATIONS_IN_SCOPE; - LOG_DEBUG(log, "Stop worked in {}", pool.name); + LOG_DEBUG(log, "Stop worker in {}", pool.name); }); } if (--pool.workers == 0) From 1264b5337e47619a692e1a6fae0d97cf4370f122 Mon Sep 17 00:00:00 2001 From: Alexander Sapin Date: Mon, 18 Mar 2024 20:57:14 +0100 Subject: [PATCH 0207/1165] More diagnostics --- src/Storages/StorageReplicatedMergeTree.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index c89be57dc07..7a63c2ee8c0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -613,18 +613,26 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( break; } else if (mutation_pointer_value >= mutation_id) /// Maybe we already processed more fresh mutation + { + LOG_TRACE(log, "Mutation {} is done because mutation pointer is {}", mutation_id, mutation_pointer_value); break; /// (numbers like 0000000000 and 0000000001) - - /// Replica can become inactive, so wait with timeout, if nothing happened -> recheck it - if (!wait_event->tryWait(1000)) - continue; + } /// Here we check mutation for errors on local replica. If they happen on this replica /// they will happen on each replica, so we can check only in-memory info. auto mutation_status = queue.getIncompleteMutationsStatus(mutation_id); /// If mutation status is empty, than local replica may just not loaded it into memory. - if (mutation_status && !mutation_status->latest_fail_reason.empty()) + if (mutation_status && (mutation_status->is_done || !mutation_status->latest_fail_reason.empty())) + { + LOG_DEBUG(log, "Mutation {} is done {} or falied {} (status: '{}')", mutation_id, mutation_status->is_done, !mutation_status->latest_fail_reason.empty(), mutation_status->latest_fail_reason); break; + } + + /// Replica can become inactive, so wait with timeout, if nothing happened -> recheck it + if (!wait_event->tryWait(1000)) + { + LOG_TRACE(log, "Failed to wait for mutation '{}', will recheck", mutation_id) + } } /// This replica inactive, don't check anything From 82da2d56e9725145e2d84f36653360927357dba8 Mon Sep 17 00:00:00 2001 From: Alexander Sapin Date: Mon, 18 Mar 2024 20:58:15 +0100 Subject: [PATCH 0208/1165] Add chassert --- src/Storages/StorageReplicatedMergeTree.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 7a63c2ee8c0..a79fe768940 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -663,6 +663,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// they will happen on each replica, so we can check only in-memory info. auto mutation_status = queue.getIncompleteMutationsStatus(mutation_id, &mutation_ids); checkMutationStatus(mutation_status, mutation_ids); + chassert(mutation_status.is_done); } if (!inactive_replicas.empty()) From a3a549bc242253866272db5945c107baea0b3803 Mon Sep 17 00:00:00 2001 From: Alexander Sapin Date: Mon, 18 Mar 2024 21:03:00 +0100 Subject: [PATCH 0209/1165] Wait for update of value in memory and in zookeeper --- src/Storages/StorageReplicatedMergeTree.cpp | 22 +++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a79fe768940..39fa65967d3 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -604,7 +604,12 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( break; } + /// Here we check mutation for errors on local replica. If they happen on this replica + /// they will happen on each replica, so we can check only in-memory info. + auto mutation_status = queue.getIncompleteMutationsStatus(mutation_id); + String mutation_pointer = fs::path(zookeeper_path) / "replicas" / replica / "mutation_pointer"; + std::string mutation_pointer_value; /// Replica could be removed if (!zookeeper->tryGet(mutation_pointer, mutation_pointer_value, nullptr, wait_event)) @@ -614,15 +619,20 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( } else if (mutation_pointer_value >= mutation_id) /// Maybe we already processed more fresh mutation { - LOG_TRACE(log, "Mutation {} is done because mutation pointer is {}", mutation_id, mutation_pointer_value); - break; /// (numbers like 0000000000 and 0000000001) + bool mutation_killed_or_done_locally = !mutation_status || mutation_status->is_done; + if (mutation_killed_or_done_locally) + { + LOG_TRACE(log, "Mutation {} is done because mutation pointer is {}", mutation_id, mutation_pointer_value); + break; /// (numbers like 0000000000 and 0000000001) + } + else + { + LOG_TRACE(log, "Mutation {} is done because mutation pointer is {}, but state is not updated in memory, will wait", mutation_id, mutation_pointer_value); + } } - /// Here we check mutation for errors on local replica. If they happen on this replica - /// they will happen on each replica, so we can check only in-memory info. - auto mutation_status = queue.getIncompleteMutationsStatus(mutation_id); /// If mutation status is empty, than local replica may just not loaded it into memory. - if (mutation_status && (mutation_status->is_done || !mutation_status->latest_fail_reason.empty())) + if (mutation_status && !mutation_status->latest_fail_reason.empty()) { LOG_DEBUG(log, "Mutation {} is done {} or falied {} (status: '{}')", mutation_id, mutation_status->is_done, !mutation_status->latest_fail_reason.empty(), mutation_status->latest_fail_reason); break; From 726db7361ffd0f31f573be45c3d8573bef4e5935 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 18 Mar 2024 20:04:56 +0000 Subject: [PATCH 0210/1165] add missing method --- tests/ci/clickhouse_helper.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 637c4519d3d..7a119ee1577 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -109,6 +109,16 @@ class ClickHouseHelper: def _insert_json_str_info(self, db, table, json_str): self.insert_json_str(self.url, self.auth, db, table, json_str) + def insert_json_into(self, db, table, json_str, safe=True): + try: + self._insert_json_str_info(db, table, json_str) + except InsertException as e: + logging.error( + "Exception happened during inserting data into clickhouse: %s", e + ) + if not safe: + raise + def insert_event_into(self, db, table, event, safe=True): event_str = json.dumps(event) try: From 9af69b42668c9c4363af1865e86d04453ad5ddcc Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 18 Mar 2024 21:05:09 +0100 Subject: [PATCH 0211/1165] fix read_rows count with external group by --- src/Processors/ISource.h | 2 +- src/Processors/Transforms/AggregatingTransform.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Processors/ISource.h b/src/Processors/ISource.h index 8c140d0d0a3..778042c5b56 100644 --- a/src/Processors/ISource.h +++ b/src/Processors/ISource.h @@ -44,7 +44,7 @@ public: void setStorageLimits(const std::shared_ptr & storage_limits_) override; /// Default implementation for all the sources. - std::optional getReadProgress() final; + std::optional getReadProgress() override; void addTotalRowsApprox(size_t value); void addTotalBytes(size_t value); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 74da97f2199..ea5c525d5f2 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -80,6 +80,8 @@ namespace return convertToChunk(block); } + std::optional getReadProgress() override { return std::nullopt; } + private: TemporaryFileStream * tmp_stream; }; From 5ac15574b72a44d349f3bfef400263893eb73457 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 18 Mar 2024 18:52:01 +0000 Subject: [PATCH 0212/1165] do not load marks on merges --- src/Storages/MergeTree/MarkRange.cpp | 5 ++ src/Storages/MergeTree/MarkRange.h | 1 + src/Storages/MergeTree/MergeTreeIOSettings.h | 2 + .../MergeTree/MergeTreeIndexReader.cpp | 3 + .../MergeTree/MergeTreeMarksLoader.cpp | 8 +++ src/Storages/MergeTree/MergeTreeMarksLoader.h | 1 + .../MergeTree/MergeTreeReaderCompact.cpp | 1 + .../MergeTree/MergeTreeReaderStream.cpp | 72 ++++++++++++++----- .../MergeTree/MergeTreeReaderStream.h | 38 +++++++--- .../MergeTree/MergeTreeReaderWide.cpp | 30 ++++++-- .../MergeTree/MergeTreeSequentialSource.cpp | 10 +-- .../test_merge_tree_load_marks/__init__.py | 0 .../configs/config.xml | 12 ++++ .../test_merge_tree_load_marks/test.py | 62 ++++++++++++++++ 14 files changed, 206 insertions(+), 39 deletions(-) create mode 100644 tests/integration/test_merge_tree_load_marks/__init__.py create mode 100644 tests/integration/test_merge_tree_load_marks/configs/config.xml create mode 100644 tests/integration/test_merge_tree_load_marks/test.py diff --git a/src/Storages/MergeTree/MarkRange.cpp b/src/Storages/MergeTree/MarkRange.cpp index bd8546f04cc..c6e98b4e5a1 100644 --- a/src/Storages/MergeTree/MarkRange.cpp +++ b/src/Storages/MergeTree/MarkRange.cpp @@ -81,6 +81,11 @@ size_t MarkRanges::getNumberOfMarks() const return result; } +bool MarkRanges::isOneRangeForWholePart(size_t num_marks_in_part) const +{ + return size() == 1 && front().begin == 0 && front().end == num_marks_in_part; +} + void MarkRanges::serialize(WriteBuffer & out) const { writeBinaryLittleEndian(this->size(), out); diff --git a/src/Storages/MergeTree/MarkRange.h b/src/Storages/MergeTree/MarkRange.h index f31d6a2a73b..9d5eac24fab 100644 --- a/src/Storages/MergeTree/MarkRange.h +++ b/src/Storages/MergeTree/MarkRange.h @@ -36,6 +36,7 @@ struct MarkRanges : public std::deque using std::deque::deque; size_t getNumberOfMarks() const; + bool isOneRangeForWholePart(size_t num_marks_in_part) const; void serialize(WriteBuffer & out) const; String describe() const; diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index feeb1808a6f..9b5842c24c9 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -44,6 +44,8 @@ struct MergeTreeReaderSettings bool enable_multiple_prewhere_read_steps = false; /// If true, try to lower size of read buffer according to granule size and compressed block size. bool adjust_read_buffer_size = true; + /// TODO: + bool always_load_marks = true; }; struct MergeTreeWriterSettings diff --git a/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 6012994b46d..e7ae1fc5c13 100644 --- a/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -31,6 +31,8 @@ std::unique_ptr makeIndexReader( load_marks_threadpool, /*num_columns_in_mark=*/ 1); + marks_loader->startAsyncLoad(); + return std::make_unique( part->getDataPartStoragePtr(), index->getFileName(), extension, marks_count, @@ -65,6 +67,7 @@ MergeTreeIndexReader::MergeTreeIndexReader( mark_cache, uncompressed_cache, std::move(settings)); + version = index_format.version; stream->adjustRightMark(getLastMark(all_mark_ranges_)); diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index a7a635f3b72..6798f97e494 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -64,6 +64,10 @@ MergeTreeMarksLoader::MergeTreeMarksLoader( , read_settings(read_settings_) , num_columns_in_mark(num_columns_in_mark_) , load_marks_threadpool(load_marks_threadpool_) +{ +} + +void MergeTreeMarksLoader::startAsyncLoad() { if (load_marks_threadpool) future = loadMarksAsync(); @@ -102,6 +106,8 @@ MergeTreeMarksGetterPtr MergeTreeMarksLoader::loadMarks() MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() { + LOG_TEST(getLogger("MergeTreeMarksLoader"), "Loading marks from path {}", mrk_path); + /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; @@ -218,7 +224,9 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksSync() } } else + { loaded_marks = loadMarksImpl(); + } if (!loaded_marks) { diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index 73dd462f2fa..2aa4474e1c5 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -50,6 +50,7 @@ public: ~MergeTreeMarksLoader(); + void startAsyncLoad(); MergeTreeMarksGetterPtr loadMarks(); size_t getNumColumns() const { return num_columns_in_mark; } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index a22bff6b8d2..8810491b62e 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -48,6 +48,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( , profile_callback(profile_callback_) , clock_type(clock_type_) { + marks_loader->startAsyncLoad(); } void MergeTreeReaderCompact::fillColumnPositions() diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 40a16176c69..15ef02440cb 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -13,6 +13,7 @@ namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; extern const int CANNOT_READ_ALL_DATA; + extern const int LOGICAL_ERROR; } MergeTreeReaderStream::MergeTreeReaderStream( @@ -41,14 +42,17 @@ MergeTreeReaderStream::MergeTreeReaderStream( { } +void MergeTreeReaderStream::loadMarks() +{ + if (!marks_getter) + marks_getter = marks_loader->loadMarks(); +} + void MergeTreeReaderStream::init() { if (initialized) return; - initialized = true; - marks_getter = marks_loader->loadMarks(); - /// Compute the size of the buffer. auto [max_mark_range_bytes, sum_mark_range_bytes] = estimateMarkRangeBytes(all_mark_ranges); @@ -110,11 +114,15 @@ void MergeTreeReaderStream::init() data_buffer = non_cached_buffer.get(); compressed_data_buffer = non_cached_buffer.get(); } + + initialized = true; } void MergeTreeReaderStream::seekToMarkAndColumn(size_t row_index, size_t column_position) { init(); + loadMarks(); + const auto & mark = marks_getter->getMark(row_index, column_position); try @@ -193,7 +201,7 @@ CompressedReadBufferBase * MergeTreeReaderStream::getCompressedDataBuffer() return compressed_data_buffer; } -size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) const +size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) { /// NOTE: if we are reading the whole file, then right_mark == marks_count /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. @@ -202,7 +210,8 @@ size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) cons if (marks_count == 0) return 0; - assert(right_mark <= marks_count); + chassert(right_mark <= marks_count); + loadMarks(); if (right_mark == 0) return marks_getter->getMark(right_mark, 0).offset_in_compressed_file; @@ -281,9 +290,9 @@ size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) cons return file_size; } -std::pair MergeTreeReaderStreamSingleColumn::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +std::pair MergeTreeReaderStreamSingleColumn::estimateMarkRangeBytes(const MarkRanges & mark_ranges) { - assert(marks_getter != nullptr); + loadMarks(); size_t max_range_bytes = 0; size_t sum_range_bytes = 0; @@ -302,7 +311,34 @@ std::pair MergeTreeReaderStreamSingleColumn::estimateMarkRangeBy return {max_range_bytes, sum_range_bytes}; } -size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) const +size_t MergeTreeReaderStreamSingleColumnWholePart::getRightOffset(size_t right_mark) +{ + if (right_mark != marks_count) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Expected one right mark: {}, got: {}", + marks_count, right_mark); + } + return file_size; +} + +std::pair MergeTreeReaderStreamSingleColumnWholePart::estimateMarkRangeBytes(const MarkRanges & mark_ranges) +{ + if (!mark_ranges.isOneRangeForWholePart(marks_count)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Expected one mark range that covers the whole part, got: {}", + mark_ranges.describe()); + } + return {file_size, file_size}; +} + +void MergeTreeReaderStreamSingleColumnWholePart::seekToMark(size_t) +{ + throw Exception(ErrorCodes::LOGICAL_ERROR, "MergeTreeReaderStreamSingleColumnWholePart cannot seek to marks"); +} + +size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) { /// NOTE: if we are reading the whole file, then right_mark == marks_count /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. @@ -311,7 +347,8 @@ size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t righ if (marks_count == 0) return 0; - assert(right_mark_non_included <= marks_count); + chassert(right_mark_non_included <= marks_count); + loadMarks(); if (right_mark_non_included == 0) return marks_getter->getMark(right_mark_non_included, column_position).offset_in_compressed_file; @@ -347,9 +384,9 @@ size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t righ } std::pair -MergeTreeReaderStreamMultipleColumns::estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) const +MergeTreeReaderStreamMultipleColumns::estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) { - assert(marks_getter != nullptr); + loadMarks(); /// As a maximal range we return the maximal size of a whole stripe. size_t max_range_bytes = 0; @@ -386,8 +423,9 @@ MergeTreeReaderStreamMultipleColumns::estimateMarkRangeBytesOneColumn(const Mark return {max_range_bytes, sum_range_bytes}; } -MarkInCompressedFile MergeTreeReaderStreamMultipleColumns::getStartOfNextStripeMark(size_t row_index, size_t column_position) const +MarkInCompressedFile MergeTreeReaderStreamMultipleColumns::getStartOfNextStripeMark(size_t row_index, size_t column_position) { + loadMarks(); const auto & current_mark = marks_getter->getMark(row_index, column_position); if (marks_getter->getNumColumns() == 1) @@ -434,27 +472,27 @@ MarkInCompressedFile MergeTreeReaderStreamMultipleColumns::getStartOfNextStripeM return marks_getter->getMark(mark_index + 1, column_position + 1); } -size_t MergeTreeReaderStreamOneOfMultipleColumns::getRightOffset(size_t right_mark_non_included) const +size_t MergeTreeReaderStreamOneOfMultipleColumns::getRightOffset(size_t right_mark_non_included) { return getRightOffsetOneColumn(right_mark_non_included, column_position); } -std::pair MergeTreeReaderStreamOneOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +std::pair MergeTreeReaderStreamOneOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) { return estimateMarkRangeBytesOneColumn(mark_ranges, column_position); } -size_t MergeTreeReaderStreamAllOfMultipleColumns::getRightOffset(size_t right_mark_non_included) const +size_t MergeTreeReaderStreamAllOfMultipleColumns::getRightOffset(size_t right_mark_non_included) { return getRightOffsetOneColumn(right_mark_non_included, marks_loader->getNumColumns() - 1); } -std::pair MergeTreeReaderStreamAllOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +std::pair MergeTreeReaderStreamAllOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) { size_t max_range_bytes = 0; size_t sum_range_bytes = 0; - for (size_t i = 0; i < marks_getter->getNumColumns(); ++i) + for (size_t i = 0; i < marks_loader->getNumColumns(); ++i) { auto [current_max, current_sum] = estimateMarkRangeBytesOneColumn(mark_ranges, i); diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index f3ca6953ceb..05341cd8acc 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -40,6 +40,7 @@ public: /// Seeks to exact mark in file. void seekToMarkAndColumn(size_t row_index, size_t column_position); + /// Seeks to the start of the file. void seekToStart(); /** @@ -53,11 +54,11 @@ public: private: /// Returns offset in file up to which it's needed to read file to read all rows up to @right_mark mark. - virtual size_t getRightOffset(size_t right_mark) const = 0; + virtual size_t getRightOffset(size_t right_mark) = 0; /// Returns estimated max amount of bytes to read among mark ranges (which is used as size for read buffer) /// and total amount of bytes to read in all mark ranges. - virtual std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const = 0; + virtual std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) = 0; const ReadBufferFromFileBase::ProfileCallback profile_callback; const clockid_t clock_type; @@ -80,6 +81,7 @@ private: protected: void init(); + void loadMarks(); const MergeTreeReaderSettings settings; const size_t marks_count; @@ -100,11 +102,25 @@ public: { } - size_t getRightOffset(size_t right_mark_non_included) const override; - std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + size_t getRightOffset(size_t right_mark_non_included) override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) override; void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, 0); } }; +class MergeTreeReaderStreamSingleColumnWholePart : public MergeTreeReaderStream +{ +public: + template + explicit MergeTreeReaderStreamSingleColumnWholePart(Args &&... args) + : MergeTreeReaderStream{std::forward(args)...} + { + } + + size_t getRightOffset(size_t right_mark_non_included) override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) override; + void seekToMark(size_t row_index) override; +}; + /// Base class for reading from file that contains multiple columns. /// It is used to read from compact parts. /// See more details about data layout in MergeTreeDataPartCompact.h. @@ -118,9 +134,9 @@ public: } protected: - size_t getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) const; - std::pair estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) const; - MarkInCompressedFile getStartOfNextStripeMark(size_t row_index, size_t column_position) const; + size_t getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position); + std::pair estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position); + MarkInCompressedFile getStartOfNextStripeMark(size_t row_index, size_t column_position); }; /// Class for reading a single column from file that contains multiple columns @@ -135,8 +151,8 @@ public: { } - size_t getRightOffset(size_t right_mark_non_included) const override; - std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + size_t getRightOffset(size_t right_mark_non_included) override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) override; void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, column_position); } private: @@ -154,8 +170,8 @@ public: { } - size_t getRightOffset(size_t right_mark_non_included) const override; - std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + size_t getRightOffset(size_t right_mark_non_included) override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) override; void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, 0); } }; diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 394a22835f1..afed94a5bb4 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -227,12 +227,13 @@ void MergeTreeReaderWide::addStreams( auto context = data_part_info_for_read->getContext(); auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; + size_t num_marks_in_part = data_part_info_for_read->getMarksCount(); auto marks_loader = std::make_shared( data_part_info_for_read, mark_cache, data_part_info_for_read->getIndexGranularityInfo().getMarksFilePath(*stream_name), - data_part_info_for_read->getMarksCount(), + num_marks_in_part, data_part_info_for_read->getIndexGranularityInfo(), settings.save_marks_in_cache, settings.read_settings, @@ -243,11 +244,26 @@ void MergeTreeReaderWide::addStreams( auto stream_settings = settings; stream_settings.is_low_cardinality_dictionary = substream_path.size() > 1 && substream_path[substream_path.size() - 2].type == ISerialization::Substream::Type::DictionaryKeys; - streams.emplace(*stream_name, std::make_unique( - data_part_info_for_read->getDataPartStorage(), *stream_name, DATA_FILE_EXTENSION, - data_part_info_for_read->getMarksCount(), all_mark_ranges, stream_settings, - uncompressed_cache, data_part_info_for_read->getFileSizeOrZero(*stream_name + DATA_FILE_EXTENSION), - std::move(marks_loader), profile_callback, clock_type)); + auto create_stream = [&]() + { + return std::make_unique( + data_part_info_for_read->getDataPartStorage(), *stream_name, DATA_FILE_EXTENSION, + num_marks_in_part, all_mark_ranges, stream_settings, + uncompressed_cache, data_part_info_for_read->getFileSizeOrZero(*stream_name + DATA_FILE_EXTENSION), + std::move(marks_loader), profile_callback, clock_type); + }; + + LOG_DEBUG(getLogger("KEK"), "settings.always_load_marks: {}, one range: {}", settings.always_load_marks, all_mark_ranges.isOneRangeForWholePart(num_marks_in_part)); + + if (!settings.always_load_marks && all_mark_ranges.isOneRangeForWholePart(num_marks_in_part)) + { + streams.emplace(*stream_name, create_stream.operator()()); + } + else + { + marks_loader->startAsyncLoad(); + streams.emplace(*stream_name, create_stream.operator()()); + } }; serialization->enumerateStreams(callback); @@ -284,7 +300,7 @@ static ReadBuffer * getStream( if (seek_to_start) stream.seekToStart(); - else if (seek_to_mark) + else if (seek_to_mark && from_mark != 0) stream.seekToMark(from_mark); return stream.getDataBuffer(); diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 81eb166b300..be7df7b4611 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -174,6 +174,7 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( .read_settings = read_settings, .save_marks_in_cache = false, .apply_deleted_mask = apply_deleted_mask, + .always_load_marks = false, }; if (!mark_ranges) @@ -184,12 +185,12 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( storage_snapshot, *mark_ranges, /*virtual_fields=*/ {}, - /*uncompressed_cache=*/{}, + /*uncompressed_cache=*/ {}, mark_cache.get(), alter_conversions, reader_settings, - {}, - {}); + /*avg_value_size_hints=*/ {}, + /*profile_callback=*/ {}); } static void fillBlockNumberColumns( @@ -219,6 +220,7 @@ try const auto & header = getPort().getHeader(); /// Part level is useful for next step for merging non-merge tree table bool add_part_level = storage.merging_params.mode != MergeTreeData::MergingParams::Ordinary; + size_t num_marks_in_part = data_part->getMarksCount(); if (!isCancelled() && current_row < data_part->rows_count) { @@ -227,7 +229,7 @@ try const auto & sample = reader->getColumns(); Columns columns(sample.size()); - size_t rows_read = reader->readRows(current_mark, data_part->getMarksCount(), continue_reading, rows_to_read, columns); + size_t rows_read = reader->readRows(current_mark, num_marks_in_part, continue_reading, rows_to_read, columns); if (rows_read) { diff --git a/tests/integration/test_merge_tree_load_marks/__init__.py b/tests/integration/test_merge_tree_load_marks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_merge_tree_load_marks/configs/config.xml b/tests/integration/test_merge_tree_load_marks/configs/config.xml new file mode 100644 index 00000000000..1c9ee8d698f --- /dev/null +++ b/tests/integration/test_merge_tree_load_marks/configs/config.xml @@ -0,0 +1,12 @@ + + + system + text_log
+ 7500 + 1048576 + 8192 + 524288 + false + test + + diff --git a/tests/integration/test_merge_tree_load_marks/test.py b/tests/integration/test_merge_tree_load_marks/test.py new file mode 100644 index 00000000000..b066b2a6ec0 --- /dev/null +++ b/tests/integration/test_merge_tree_load_marks/test.py @@ -0,0 +1,62 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node", + main_configs=["configs/config.xml"], +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +# This test is bad and it should be a functional test but S3 metrics +# are accounted incorrectly for merges in part_log and query_log. +# Also we have text_log with level 'trace' in functional tests +# but this test requeires text_log with level 'test'. + + +@pytest.mark.parametrize("min_bytes_for_wide_part", [0, 1000000000]) +def test_merge_load_marks(started_cluster, min_bytes_for_wide_part): + node.query( + f""" + DROP TABLE IF EXISTS t_load_marks; + + CREATE TABLE t_load_marks (a UInt64, b UInt64) + ENGINE = MergeTree ORDER BY a + SETTINGS min_bytes_for_wide_part = {min_bytes_for_wide_part}; + + INSERT INTO t_load_marks SELECT number, number FROM numbers(1000); + INSERT INTO t_load_marks SELECT number, number FROM numbers(1000); + + OPTIMIZE TABLE t_load_marks FINAL; + SYSTEM FLUSH LOGS; + """ + ) + + uuid = node.query( + "SELECT uuid FROM system.tables WHERE table = 't_prewarm_merge'" + ).strip() + + result = node.query( + f""" + SELECT count() + FROM system.text_log + WHERE (query_id LIKE '%{uuid}::all_1_2_1%') AND (message LIKE '%Loading marks%') + """ + ).strip() + + result = int(result) + + is_wide = min_bytes_for_wide_part == 0 + not_loaded = result == 0 + + assert is_wide == not_loaded From 306f642d97ce51bed6fda59cac1c3e3465f6ef5a Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 17 Mar 2024 11:57:01 +0100 Subject: [PATCH 0213/1165] Update dictionary's comment in the same safe manner too. --- src/Dictionaries/DictionaryFactory.cpp | 6 +----- src/Dictionaries/IDictionary.h | 14 ++++++++------ .../ExternalDictionariesLoader.cpp | 13 +++++++++---- src/Storages/StorageDictionary.cpp | 18 +++++++++--------- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index a566fb27de4..337121822ea 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -55,11 +55,7 @@ DictionaryPtr DictionaryFactory::create( if (found != registered_layouts.end()) { const auto & layout_creator = found->second.layout_create_function; - auto result = layout_creator(name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); - if (config.hasProperty(config_prefix + ".comment")) - result->setDictionaryComment(config.getString(config_prefix + ".comment")); - - return result; + return layout_creator(name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); } } diff --git a/src/Dictionaries/IDictionary.h b/src/Dictionaries/IDictionary.h index 9770e93669e..bab80d3cd57 100644 --- a/src/Dictionaries/IDictionary.h +++ b/src/Dictionaries/IDictionary.h @@ -332,12 +332,6 @@ public: return std::static_pointer_cast(IExternalLoadable::shared_from_this()); } - void setDictionaryComment(String new_comment) - { - std::lock_guard lock{mutex}; - dictionary_comment = std::move(new_comment); - } - String getDictionaryComment() const { std::lock_guard lock{mutex}; @@ -454,6 +448,14 @@ public: dictionary_id = new_dictionary_id; } + /// Internally called by ExternalDictionariesLoader. + /// In order to update the dictionary comment change its configuration first and then call ExternalDictionariesLoader::reloadConfig(). + void updateDictionaryComment(const String & new_dictionary_comment) + { + std::lock_guard lock{mutex}; + dictionary_comment = new_dictionary_comment; + } + private: mutable std::mutex mutex; StorageID dictionary_id TSA_GUARDED_BY(mutex); diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 0591a6232d0..68f22df2947 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -45,14 +45,17 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( bool ExternalDictionariesLoader::doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & old_config, const String & old_key_in_config, const Poco::Util::AbstractConfiguration & new_config, const String & new_key_in_config) const { + std::unordered_set ignore_keys; + ignore_keys.insert("comment"); /// We always can change the comment without reloading a dictionary. + /// If the database is atomic then a dictionary can be renamed without reloading. if (!old_config.getString(old_key_in_config + ".uuid", "").empty() && !new_config.getString(new_key_in_config + ".uuid", "").empty()) { - static const std::unordered_set ignore_keys{"name", "database"}; - bool only_name_or_database_may_differ = isSameConfigurationIgnoringKeys(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); - return !only_name_or_database_may_differ; + ignore_keys.insert("name"); + ignore_keys.insert("database"); } - return true; + + return !isSameConfigurationIgnoringKeys(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); } void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const @@ -67,6 +70,8 @@ void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExterna if ((new_dictionary_id.uuid == old_dictionary_id.uuid) && (new_dictionary_id.uuid != UUIDHelpers::Nil)) dict.updateDictionaryID(new_dictionary_id); } + + dict.updateDictionaryComment(config.getString(key_in_config + ".comment", "")); } ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::getDictionary(const std::string & dictionary_name, ContextPtr local_context) const diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 414c66e9522..a0c4156a704 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -278,19 +278,19 @@ void StorageDictionary::alter(const AlterCommands & params, ContextPtr alter_con auto new_comment = getInMemoryMetadataPtr()->comment; - auto storage_id = getStorageID(); - const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); - auto result = external_dictionaries_loader.getLoadResult(storage_id.getInternalDictionaryName()); + /// It's better not to update an associated `IDictionary` directly here because it can be not loaded yet or + /// it can be in the process of loading or reloading right now. + /// The correct way is to update the dictionary's configuration first and then ask ExternalDictionariesLoader to reload our dictionary. - if (result.object) { - auto dictionary = std::static_pointer_cast(result.object); - auto * dictionary_non_const = const_cast(dictionary.get()); - dictionary_non_const->setDictionaryComment(new_comment); + std::lock_guard lock(dictionary_config_mutex); + auto new_configuration = ConfigHelper::clone(*configuration); + new_configuration->setString("dictionary.comment", new_comment); + configuration = new_configuration; } - std::lock_guard lock(dictionary_config_mutex); - configuration->setString("dictionary.comment", new_comment); + const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); + external_dictionaries_loader.reloadConfig(getStorageID().getInternalDictionaryName()); } void registerStorageDictionary(StorageFactory & factory) From c8375cd16777010a35d11340bf7b8c9c60c1fa1e Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 18 Mar 2024 16:44:55 +0100 Subject: [PATCH 0214/1165] Better code: remove unnecessary usage of const_cast and std::function. --- src/Dictionaries/CacheDictionary.h | 2 +- src/Dictionaries/DirectDictionary.h | 2 +- src/Dictionaries/FlatDictionary.h | 2 +- src/Dictionaries/HashedArrayDictionary.h | 2 +- src/Dictionaries/HashedDictionary.h | 2 +- src/Dictionaries/IPAddressDictionary.h | 2 +- .../PolygonDictionaryImplementations.cpp | 6 +- .../PolygonDictionaryImplementations.h | 6 +- src/Dictionaries/RangeHashedDictionary.h | 2 +- src/Dictionaries/RegExpTreeDictionary.h | 2 +- ...alUserDefinedExecutableFunctionsLoader.cpp | 2 +- ...rnalUserDefinedExecutableFunctionsLoader.h | 2 +- .../UserDefinedExecutableFunction.h | 2 +- .../ExternalDictionariesLoader.cpp | 2 +- src/Interpreters/ExternalDictionariesLoader.h | 4 +- src/Interpreters/ExternalLoader.cpp | 60 +++++-------------- src/Interpreters/ExternalLoader.h | 7 ++- src/Interpreters/IExternalLoadable.h | 2 +- 18 files changed, 41 insertions(+), 68 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.h b/src/Dictionaries/CacheDictionary.h index 8897fb40fa9..50f59e3eddb 100644 --- a/src/Dictionaries/CacheDictionary.h +++ b/src/Dictionaries/CacheDictionary.h @@ -98,7 +98,7 @@ public: bool supportUpdates() const override { return false; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared( getDictionaryID(), diff --git a/src/Dictionaries/DirectDictionary.h b/src/Dictionaries/DirectDictionary.h index 8bf183fd1ab..d4b4cd8e698 100644 --- a/src/Dictionaries/DirectDictionary.h +++ b/src/Dictionaries/DirectDictionary.h @@ -50,7 +50,7 @@ public: double getLoadFactor() const override { return 0; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone()); } diff --git a/src/Dictionaries/FlatDictionary.h b/src/Dictionaries/FlatDictionary.h index 231819ab4ee..7b00ce57455 100644 --- a/src/Dictionaries/FlatDictionary.h +++ b/src/Dictionaries/FlatDictionary.h @@ -57,7 +57,7 @@ public: double getLoadFactor() const override { return static_cast(element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone(), configuration, update_field_loaded_block); } diff --git a/src/Dictionaries/HashedArrayDictionary.h b/src/Dictionaries/HashedArrayDictionary.h index 4b2570ad928..d4bb67b9606 100644 --- a/src/Dictionaries/HashedArrayDictionary.h +++ b/src/Dictionaries/HashedArrayDictionary.h @@ -73,7 +73,7 @@ public: double getLoadFactor() const override { return static_cast(total_element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared>(getDictionaryID(), dict_struct, source_ptr->clone(), configuration, update_field_loaded_block); } diff --git a/src/Dictionaries/HashedDictionary.h b/src/Dictionaries/HashedDictionary.h index 37812de29d9..4080c018289 100644 --- a/src/Dictionaries/HashedDictionary.h +++ b/src/Dictionaries/HashedDictionary.h @@ -116,7 +116,7 @@ public: double getLoadFactor() const override { return static_cast(element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared>( getDictionaryID(), diff --git a/src/Dictionaries/IPAddressDictionary.h b/src/Dictionaries/IPAddressDictionary.h index cebdefcf7ae..bdd02157077 100644 --- a/src/Dictionaries/IPAddressDictionary.h +++ b/src/Dictionaries/IPAddressDictionary.h @@ -57,7 +57,7 @@ public: double getLoadFactor() const override { return static_cast(element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone(), configuration); } diff --git a/src/Dictionaries/PolygonDictionaryImplementations.cpp b/src/Dictionaries/PolygonDictionaryImplementations.cpp index 3feca2ec410..64d29458430 100644 --- a/src/Dictionaries/PolygonDictionaryImplementations.cpp +++ b/src/Dictionaries/PolygonDictionaryImplementations.cpp @@ -29,7 +29,7 @@ PolygonDictionarySimple::PolygonDictionarySimple( { } -std::shared_ptr PolygonDictionarySimple::clone() const +std::shared_ptr PolygonDictionarySimple::clone() const { return std::make_shared( this->getDictionaryID(), @@ -76,7 +76,7 @@ PolygonDictionaryIndexEach::PolygonDictionaryIndexEach( } } -std::shared_ptr PolygonDictionaryIndexEach::clone() const +std::shared_ptr PolygonDictionaryIndexEach::clone() const { return std::make_shared( this->getDictionaryID(), @@ -126,7 +126,7 @@ PolygonDictionaryIndexCell::PolygonDictionaryIndexCell( { } -std::shared_ptr PolygonDictionaryIndexCell::clone() const +std::shared_ptr PolygonDictionaryIndexCell::clone() const { return std::make_shared( this->getDictionaryID(), diff --git a/src/Dictionaries/PolygonDictionaryImplementations.h b/src/Dictionaries/PolygonDictionaryImplementations.h index 912d501bcde..690ff3a0f1b 100644 --- a/src/Dictionaries/PolygonDictionaryImplementations.h +++ b/src/Dictionaries/PolygonDictionaryImplementations.h @@ -23,7 +23,7 @@ public: DictionaryLifetime dict_lifetime_, Configuration configuration_); - std::shared_ptr clone() const override; + std::shared_ptr clone() const override; private: bool find(const Point & point, size_t & polygon_index) const override; @@ -47,7 +47,7 @@ public: int min_intersections_, int max_depth_); - std::shared_ptr clone() const override; + std::shared_ptr clone() const override; static constexpr size_t kMinIntersectionsDefault = 1; static constexpr size_t kMaxDepthDefault = 5; @@ -75,7 +75,7 @@ public: size_t min_intersections_, size_t max_depth_); - std::shared_ptr clone() const override; + std::shared_ptr clone() const override; static constexpr size_t kMinIntersectionsDefault = 1; static constexpr size_t kMaxDepthDefault = 5; diff --git a/src/Dictionaries/RangeHashedDictionary.h b/src/Dictionaries/RangeHashedDictionary.h index 1a6ee7e81d4..9f7dc5dc68a 100644 --- a/src/Dictionaries/RangeHashedDictionary.h +++ b/src/Dictionaries/RangeHashedDictionary.h @@ -101,7 +101,7 @@ public: double getLoadFactor() const override { return static_cast(element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { auto result = std::make_shared( getDictionaryID(), diff --git a/src/Dictionaries/RegExpTreeDictionary.h b/src/Dictionaries/RegExpTreeDictionary.h index 9e14abb49d0..a361c68879e 100644 --- a/src/Dictionaries/RegExpTreeDictionary.h +++ b/src/Dictionaries/RegExpTreeDictionary.h @@ -86,7 +86,7 @@ public: bool hasHierarchy() const override { return false; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared( getDictionaryID(), structure, source_ptr->clone(), configuration, use_vectorscan, flag_case_insensitive, flag_dotall); diff --git a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp index db98f88e53b..a4f17aa1201 100644 --- a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp +++ b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp @@ -120,7 +120,7 @@ void ExternalUserDefinedExecutableFunctionsLoader::reloadFunction(const std::str loadOrReload(user_defined_function_name); } -ExternalLoader::LoadablePtr ExternalUserDefinedExecutableFunctionsLoader::create(const std::string & name, +ExternalLoader::LoadableMutablePtr ExternalUserDefinedExecutableFunctionsLoader::createObject(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string &) const diff --git a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h index 1a62175eb0c..eb86986c391 100644 --- a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h +++ b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h @@ -28,7 +28,7 @@ public: void reloadFunction(const std::string & user_defined_function_name) const; protected: - LoadablePtr create(const std::string & name, + LoadableMutablePtr createObject(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string & repository_name) const override; diff --git a/src/Functions/UserDefined/UserDefinedExecutableFunction.h b/src/Functions/UserDefined/UserDefinedExecutableFunction.h index 989f9dfe895..d48be215c7d 100644 --- a/src/Functions/UserDefined/UserDefinedExecutableFunction.h +++ b/src/Functions/UserDefined/UserDefinedExecutableFunction.h @@ -62,7 +62,7 @@ public: return true; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared(configuration, coordinator, lifetime); } diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 68f22df2947..34ad240d089 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -32,7 +32,7 @@ ExternalDictionariesLoader::ExternalDictionariesLoader(ContextPtr global_context enablePeriodicUpdates(true); } -ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( +ExternalLoader::LoadableMutablePtr ExternalDictionariesLoader::createObject( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string & repository_name) const { diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h index f3aa8fa5f8e..0fc58aa1edf 100644 --- a/src/Interpreters/ExternalDictionariesLoader.h +++ b/src/Interpreters/ExternalDictionariesLoader.h @@ -40,8 +40,8 @@ public: static void resetAll(); protected: - LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, - const std::string & key_in_config, const std::string & repository_name) const override; + LoadableMutablePtr createObject(const std::string & name, const Poco::Util::AbstractConfiguration & config, + const std::string & key_in_config, const std::string & repository_name) const override; bool doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & old_config, const String & old_key_in_config, const Poco::Util::AbstractConfiguration & new_config, const String & new_key_in_config) const override; diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 6ab083f9fb7..bd56a540128 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -87,9 +87,6 @@ namespace lock = std::unique_lock(mutex); } }; - - using DoesConfigChangeRequiresReloadingObjectFunction = std::function; - using UpdateObjectFromConfigWithoutReloadingFunction = std::function; } @@ -98,10 +95,7 @@ namespace class ExternalLoader::LoadablesConfigReader : private boost::noncopyable { public: - LoadablesConfigReader(const String & type_name_, LoggerPtr log_) - : type_name(type_name_), log(log_) - { - } + LoadablesConfigReader(const String & type_name_, LoggerPtr log_) : type_name(type_name_), log(log_) { } ~LoadablesConfigReader() = default; using Repository = IExternalLoaderConfigRepository; @@ -397,21 +391,8 @@ private: class ExternalLoader::LoadingDispatcher : private boost::noncopyable { public: - /// Called to load or reload an object. - using CreateObjectFunction = std::function; - - LoadingDispatcher( - const CreateObjectFunction & create_object_function_, - const DoesConfigChangeRequiresReloadingObjectFunction & does_config_change_requires_reloading_object_, - const UpdateObjectFromConfigWithoutReloadingFunction & update_object_from_config_without_reloading_, - const String & type_name_, - LoggerPtr log_) - : create_object(create_object_function_) - , does_config_change_requires_reloading_object(does_config_change_requires_reloading_object_) - , update_object_from_config_without_reloading(update_object_from_config_without_reloading_) - , type_name(type_name_) - , log(log_) + LoadingDispatcher(const String & type_name_, LoggerPtr log_, const ExternalLoader & external_loader_) + : type_name(type_name_), log(log_), external_loader(external_loader_) { } @@ -471,11 +452,11 @@ public: if (config_changed) { if (info.object) - update_object_from_config_without_reloading(*info.object, *new_config->config, new_config->key_in_config); + external_loader.updateObjectFromConfigWithoutReloading(*info.object, *new_config->config, new_config->key_in_config); if (info.triedToLoad()) { - bool config_change_requires_reloading = does_config_change_requires_reloading_object(*previous_config->config, previous_config->key_in_config, *new_config->config, new_config->key_in_config); + bool config_change_requires_reloading = external_loader.doesConfigChangeRequiresReloadingObject(*previous_config->config, previous_config->key_in_config, *new_config->config, new_config->key_in_config); if (config_change_requires_reloading) { /// The object has been tried to load before, so it is currently in use or was in use @@ -786,7 +767,7 @@ private: } String name; - LoadablePtr object; + LoadableMutablePtr object; std::shared_ptr config; TimePoint loading_start_time; TimePoint loading_end_time; @@ -1046,17 +1027,17 @@ private: } /// Load one object, returns object ptr or exception. - std::pair + std::pair loadSingleObject(const String & name, const ObjectConfig & config, LoadablePtr previous_version) { /// Use `create_function` to perform the actual loading. /// It's much better to do it with `mutex` unlocked because the loading can take a lot of time /// and require access to other objects. - LoadablePtr new_object; + LoadableMutablePtr new_object; std::exception_ptr new_exception; try { - new_object = create_object(name, config, previous_version); + new_object = external_loader.createOrCloneObject(name, config, previous_version); } catch (...) { @@ -1070,7 +1051,7 @@ private: const String & name, size_t loading_id, LoadablePtr previous_version, - LoadablePtr new_object, + LoadableMutablePtr new_object, std::exception_ptr new_exception, size_t error_count, const LoadingGuardForAsyncLoad &) @@ -1134,7 +1115,7 @@ private: if (new_object) { - update_object_from_config_without_reloading(*new_object, *info->config->config, info->config->key_in_config); + external_loader.updateObjectFromConfigWithoutReloading(*new_object, *info->config->config, info->config->key_in_config); info->object = new_object; } @@ -1210,11 +1191,9 @@ private: } } - const CreateObjectFunction create_object; - const DoesConfigChangeRequiresReloadingObjectFunction does_config_change_requires_reloading_object; - const UpdateObjectFromConfigWithoutReloadingFunction update_object_from_config_without_reloading; const String type_name; - LoggerPtr log; + const LoggerPtr log; + const ExternalLoader & external_loader; mutable std::mutex mutex; std::condition_variable event; @@ -1296,14 +1275,7 @@ private: ExternalLoader::ExternalLoader(const String & type_name_, LoggerPtr log_) : config_files_reader(std::make_unique(type_name_, log_)) - , loading_dispatcher(std::make_unique( - [this](auto && a, auto && b, auto && c) { return createObject(a, b, c); }, - [this](const Poco::Util::AbstractConfiguration & config_1, const String & key_in_config_1, const Poco::Util::AbstractConfiguration & config_2, const String & key_in_config_2) - { return doesConfigChangeRequiresReloadingObject(config_1, key_in_config_1, config_2, key_in_config_2); }, - [this](const IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) - { return updateObjectFromConfigWithoutReloading(const_cast(object), config, key_in_config); }, - type_name_, - log_)) + , loading_dispatcher(std::make_unique(type_name_, log_, *this)) , periodic_updater(std::make_unique(*config_files_reader, *loading_dispatcher)) , type_name(type_name_) , log(log_) @@ -1530,13 +1502,13 @@ void ExternalLoader::reloadConfig(const String & repository_name, const String & loading_dispatcher->setConfiguration(config_files_reader->read(repository_name, path)); } -ExternalLoader::LoadablePtr ExternalLoader::createObject( +ExternalLoader::LoadableMutablePtr ExternalLoader::createOrCloneObject( const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const { if (previous_version) return previous_version->clone(); - return create(name, *config.config, config.key_in_config, config.repository_name); + return createObject(name, *config.config, config.key_in_config, config.repository_name); } template ExternalLoader::LoadablePtr ExternalLoader::getLoadResult(const String &) const; diff --git a/src/Interpreters/ExternalLoader.h b/src/Interpreters/ExternalLoader.h index 781d8ca9e7b..c746ce39b2d 100644 --- a/src/Interpreters/ExternalLoader.h +++ b/src/Interpreters/ExternalLoader.h @@ -50,6 +50,7 @@ class ExternalLoader { public: using LoadablePtr = std::shared_ptr; + using LoadableMutablePtr = std::shared_ptr; using Loadables = std::vector; using Status = ExternalLoaderStatus; @@ -211,7 +212,7 @@ public: void reloadConfig(const String & repository_name, const String & path) const; protected: - virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0; + virtual LoadableMutablePtr createObject(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0; /// Returns whether the object must be reloaded after a specified change in its configuration. virtual bool doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & /* old_config */, const String & /* old_key_in_config */, @@ -227,7 +228,7 @@ private: Strings getAllTriedToLoadNames() const; - LoadablePtr createObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const; + LoadableMutablePtr createOrCloneObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const; class LoadablesConfigReader; std::unique_ptr config_files_reader; @@ -239,7 +240,7 @@ private: std::unique_ptr periodic_updater; const String type_name; - LoggerPtr log; + const LoggerPtr log; }; } diff --git a/src/Interpreters/IExternalLoadable.h b/src/Interpreters/IExternalLoadable.h index 3c004508b0a..1076bd88027 100644 --- a/src/Interpreters/IExternalLoadable.h +++ b/src/Interpreters/IExternalLoadable.h @@ -43,7 +43,7 @@ public: /// If lifetime exceeded and isModified(), ExternalLoader replace current object with the result of clone(). virtual bool isModified() const = 0; /// Returns new object with the same configuration. Is used to update modified object when lifetime exceeded. - virtual std::shared_ptr clone() const = 0; + virtual std::shared_ptr clone() const = 0; }; } From a1a4bd3514f72b4545fbb734749dcb9d7f513007 Mon Sep 17 00:00:00 2001 From: nemonlou Date: Wed, 6 Mar 2024 10:42:30 +0800 Subject: [PATCH 0215/1165] make nulls direction configuable for FullSortingMergeJoin --- src/Core/Settings.h | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 7 +- src/Processors/QueryPlan/JoinStep.cpp | 8 +- src/Processors/QueryPlan/JoinStep.h | 4 +- .../Transforms/MergeJoinTransform.cpp | 82 ++++++++++++------- .../Transforms/MergeJoinTransform.h | 16 +++- src/QueryPipeline/QueryPipelineBuilder.cpp | 3 +- src/QueryPipeline/QueryPipelineBuilder.h | 1 + 8 files changed, 84 insertions(+), 38 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a3c5638d97f..8d48b3f5e68 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -891,6 +891,7 @@ class IColumn; M(Int64, ignore_cold_parts_seconds, 0, "Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries until they're either pre-warmed (see cache_populated_by_fetch) or this many seconds old. Only for Replicated-/SharedMergeTree.", 0) \ M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \ M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \ + M(Bool, nulls_biggest_in_smj, true, "Treat nulls as biggest in sort. Used in sort merge join for compare null keys.", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS. diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 7c87dadfce6..6f0a9fa9bfb 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1693,9 +1693,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( query_plan.getCurrentDataStream(), joined_plan->getCurrentDataStream(), expressions.join, settings.max_block_size, max_streams, - analysis_result.optimize_read_in_order); + analysis_result.optimize_read_in_order, + null_direct_hint); join_step->setStepDescription(fmt::format("JOIN {}", expressions.join->pipelineType())); std::vector plans; diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index 1931b1eb3a1..0c46ce7893d 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -44,8 +44,10 @@ JoinStep::JoinStep( JoinPtr join_, size_t max_block_size_, size_t max_streams_, - bool keep_left_read_in_order_) - : join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_) + bool keep_left_read_in_order_, + int null_direction_hint_) + : join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_), + null_direction_hint(null_direction_hint_) { updateInputStreams(DataStreams{left_stream_, right_stream_}); } @@ -58,7 +60,7 @@ QueryPipelineBuilderPtr JoinStep::updatePipeline(QueryPipelineBuilders pipelines if (join->pipelineType() == JoinPipelineType::YShaped) { auto joined_pipeline = QueryPipelineBuilder::joinPipelinesYShaped( - std::move(pipelines[0]), std::move(pipelines[1]), join, output_stream->header, max_block_size, &processors); + std::move(pipelines[0]), std::move(pipelines[1]), join, output_stream->header, max_block_size, null_direction_hint, &processors); joined_pipeline->resize(max_streams); return joined_pipeline; } diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h index a9059a083fe..08909ce48a9 100644 --- a/src/Processors/QueryPlan/JoinStep.h +++ b/src/Processors/QueryPlan/JoinStep.h @@ -19,7 +19,8 @@ public: JoinPtr join_, size_t max_block_size_, size_t max_streams_, - bool keep_left_read_in_order_); + bool keep_left_read_in_order_, + int null_direction_hint_ = 1); String getName() const override { return "Join"; } @@ -42,6 +43,7 @@ private: size_t max_block_size; size_t max_streams; bool keep_left_read_in_order; + int null_direction_hint; }; /// Special step for the case when Join is already filled. diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 2d313d4ea5c..c8e3a806a9f 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -43,7 +43,7 @@ FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns) } template -int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, size_t lhs_pos, size_t rhs_pos, int null_direction_hint = 1) +int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, size_t lhs_pos, size_t rhs_pos, int null_direction_hint) { if constexpr (has_left_nulls && has_right_nulls) { @@ -88,35 +88,36 @@ int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, } int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, size_t lpos, - const SortCursorImpl & rhs, size_t rpos) + const SortCursorImpl & rhs, size_t rpos, + int null_direction_hint) { for (size_t i = 0; i < lhs.sort_columns_size; ++i) { /// TODO(@vdimir): use nullableCompareAt only if there's nullable columns - int cmp = nullableCompareAt(*lhs.sort_columns[i], *rhs.sort_columns[i], lpos, rpos); + int cmp = nullableCompareAt(*lhs.sort_columns[i], *rhs.sort_columns[i], lpos, rpos, null_direction_hint); if (cmp != 0) return cmp; } return 0; } -int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImpl & rhs) +int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImpl & rhs, int null_direction_hint) { - return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow()); + return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow(), null_direction_hint); } -bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs) +bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint) { /// The last row of left cursor is less than the current row of the right cursor. - int cmp = compareCursors(lhs, lhs.rows - 1, rhs, rhs.getRow()); + int cmp = compareCursors(lhs, lhs.rows - 1, rhs, rhs.getRow(), null_direction_hint); return cmp < 0; } -int ALWAYS_INLINE totallyCompare(SortCursorImpl & lhs, SortCursorImpl & rhs) +int ALWAYS_INLINE totallyCompare(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint) { - if (totallyLess(lhs, rhs)) + if (totallyLess(lhs, rhs, null_direction_hint)) return -1; - if (totallyLess(rhs, lhs)) + if (totallyLess(rhs, lhs, null_direction_hint)) return 1; return 0; } @@ -270,9 +271,11 @@ bool FullMergeJoinCursor::fullyCompleted() const MergeJoinAlgorithm::MergeJoinAlgorithm( JoinPtr table_join_, const Blocks & input_headers, - size_t max_block_size_) + size_t max_block_size_, + int null_direction_hint_) : table_join(table_join_) , max_block_size(max_block_size_) + , null_direction_hint(null_direction_hint_) , log(getLogger("MergeJoinAlgorithm")) { if (input_headers.size() != 2) @@ -356,7 +359,7 @@ void MergeJoinAlgorithm::consume(Input & input, size_t source_num) cursors[source_num]->setChunk(std::move(input.chunk)); } -template +template struct AllJoinImpl { constexpr static bool enabled = isInner(kind) || isLeft(kind) || isRight(kind) || isFull(kind); @@ -382,7 +385,7 @@ struct AllJoinImpl lpos = left_cursor->getRow(); rpos = right_cursor->getRow(); - cmp = compareCursors(left_cursor.cursor, right_cursor.cursor); + cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, nullDirection(nullOrder)); if (cmp == 0) { size_t lnum = nextDistinct(left_cursor.cursor); @@ -432,19 +435,37 @@ struct AllJoinImpl } }; -template class Impl, typename ... Args> -void dispatchKind(JoinKind kind, Args && ... args) +template class Impl, typename ... Args> +void dispatchKind(JoinKind kind, int null_direction_hint, Args && ... args) { - if (Impl::enabled && kind == JoinKind::Inner) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Left) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Right) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Full) - return Impl::join(std::forward(args)...); + if (isSmall(null_direction_hint)) + { + if (Impl::enabled && kind == JoinKind::Inner) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Left) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Right) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Full) + return Impl::join(std::forward(args)...); + else + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + + } else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + { + if (Impl::enabled && kind == JoinKind::Inner) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Left) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Right) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Full) + return Impl::join(std::forward(args)...); + else + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + + } } std::optional MergeJoinAlgorithm::handleAllJoinState() @@ -517,7 +538,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind) { PaddedPODArray idx_map[2]; - dispatchKind(kind, *cursors[0], *cursors[1], max_block_size, idx_map[0], idx_map[1], all_join_state); + dispatchKind(kind, null_direction_hint, *cursors[0], *cursors[1], max_block_size, idx_map[0], idx_map[1], all_join_state); assert(idx_map[0].size() == idx_map[1].size()); Chunk result; @@ -567,7 +588,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind) } -template +template struct AnyJoinImpl { constexpr static bool enabled = isInner(kind) || isLeft(kind) || isRight(kind); @@ -599,7 +620,7 @@ struct AnyJoinImpl lpos = left_cursor->getRow(); rpos = right_cursor->getRow(); - cmp = compareCursors(left_cursor.cursor, right_cursor.cursor); + cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, nullDirection(order)); if (cmp == 0) { if constexpr (isLeftOrFull(kind)) @@ -723,7 +744,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin(JoinKind kind) PaddedPODArray idx_map[2]; size_t prev_pos[] = {current_left.getRow(), current_right.getRow()}; - dispatchKind(kind, *cursors[0], *cursors[1], idx_map[0], idx_map[1], any_join_state); + dispatchKind(kind, null_direction_hint, *cursors[0], *cursors[1], idx_map[0], idx_map[1], any_join_state); assert(idx_map[0].empty() || idx_map[1].empty() || idx_map[0].size() == idx_map[1].size()); size_t num_result_rows = std::max(idx_map[0].size(), idx_map[1].size()); @@ -816,7 +837,7 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() } /// check if blocks are not intersecting at all - if (int cmp = totallyCompare(cursors[0]->cursor, cursors[1]->cursor); cmp != 0) + if (int cmp = totallyCompare(cursors[0]->cursor, cursors[1]->cursor, null_direction_hint); cmp != 0) { if (cmp < 0) { @@ -851,6 +872,7 @@ MergeJoinTransform::MergeJoinTransform( const Blocks & input_headers, const Block & output_header, size_t max_block_size, + int null_direction_hint_, UInt64 limit_hint_) : IMergingTransform( input_headers, @@ -859,7 +881,7 @@ MergeJoinTransform::MergeJoinTransform( limit_hint_, /* always_read_till_end_= */ false, /* empty_chunk_on_finish_= */ true, - table_join, input_headers, max_block_size) + table_join, input_headers, max_block_size, null_direction_hint_) , log(getLogger("MergeJoinTransform")) { LOG_TRACE(log, "Use MergeJoinTransform"); diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 959550067f7..43485321122 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -220,6 +220,17 @@ private: bool recieved_all_blocks = false; }; +/// Join method. +enum class NullOrder +{ + SMALLEST, /// null is treated as smallest + BIGGEST /// null is treated as biggest +}; + +inline constexpr bool isSmall(int null_direction) { return null_direction == 1; } + +inline constexpr int nullDirection(NullOrder order) {return order == NullOrder::SMALLEST ? 1 : -1;} + /* * This class is used to join chunks from two sorted streams. * It is used in MergeJoinTransform. @@ -227,7 +238,8 @@ private: class MergeJoinAlgorithm final : public IMergingAlgorithm { public: - explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_); + explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_, + int null_direction_hint = 1); const char * getName() const override { return "MergeJoinAlgorithm"; } void initialize(Inputs inputs) override; @@ -258,6 +270,7 @@ private: JoinPtr table_join; size_t max_block_size; + int null_direction_hint; struct Statistic { @@ -282,6 +295,7 @@ public: const Blocks & input_headers, const Block & output_header, size_t max_block_size, + int null_direction_hint, UInt64 limit_hint = 0); String getName() const override { return "MergeJoinTransform"; } diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index 67a8fe5dcab..e338c3ce0fa 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -349,6 +349,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesYShaped JoinPtr join, const Block & out_header, size_t max_block_size, + int null_direction_hint, Processors * collected_processors) { left->checkInitializedAndNotCompleted(); @@ -376,7 +377,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesYShaped } else { - auto joining = std::make_shared(join, inputs, out_header, max_block_size); + auto joining = std::make_shared(join, inputs, out_header, max_block_size, null_direction_hint); return mergePipelines(std::move(left), std::move(right), std::move(joining), collected_processors); } } diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index f0b2ead687e..4753f957a25 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -137,6 +137,7 @@ public: JoinPtr table_join, const Block & out_header, size_t max_block_size, + int null_direction_hint, Processors * collected_processors = nullptr); /// Add other pipeline and execute it before current one. From 0b5fc743f2e0711556ab4628aecd13e5fcd1a9b8 Mon Sep 17 00:00:00 2001 From: nemonlou Date: Tue, 12 Mar 2024 09:55:02 +0800 Subject: [PATCH 0216/1165] make nulls direction configuable for FullSortingMergeJoin(fix review comments) --- src/Core/Settings.h | 1 - src/Interpreters/InterpreterSelectQuery.cpp | 7 +-- .../Transforms/MergeJoinTransform.cpp | 56 +++++++------------ .../Transforms/MergeJoinTransform.h | 11 ---- 4 files changed, 22 insertions(+), 53 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 8d48b3f5e68..a3c5638d97f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -891,7 +891,6 @@ class IColumn; M(Int64, ignore_cold_parts_seconds, 0, "Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries until they're either pre-warmed (see cache_populated_by_fetch) or this many seconds old. Only for Replicated-/SharedMergeTree.", 0) \ M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \ M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \ - M(Bool, nulls_biggest_in_smj, true, "Treat nulls as biggest in sort. Used in sort merge join for compare null keys.", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS. diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6f0a9fa9bfb..7c87dadfce6 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1693,10 +1693,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( query_plan.getCurrentDataStream(), joined_plan->getCurrentDataStream(), expressions.join, settings.max_block_size, max_streams, - analysis_result.optimize_read_in_order, - null_direct_hint); + analysis_result.optimize_read_in_order); join_step->setStepDescription(fmt::format("JOIN {}", expressions.join->pipelineType())); std::vector plans; diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index c8e3a806a9f..6288a850d76 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -359,7 +359,7 @@ void MergeJoinAlgorithm::consume(Input & input, size_t source_num) cursors[source_num]->setChunk(std::move(input.chunk)); } -template +template struct AllJoinImpl { constexpr static bool enabled = isInner(kind) || isLeft(kind) || isRight(kind) || isFull(kind); @@ -369,7 +369,8 @@ struct AllJoinImpl size_t max_block_size, PaddedPODArray & left_map, PaddedPODArray & right_map, - std::unique_ptr & state) + std::unique_ptr & state, + int null_direction_hint) { right_map.clear(); right_map.reserve(max_block_size); @@ -385,7 +386,7 @@ struct AllJoinImpl lpos = left_cursor->getRow(); rpos = right_cursor->getRow(); - cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, nullDirection(nullOrder)); + cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, null_direction_hint); if (cmp == 0) { size_t lnum = nextDistinct(left_cursor.cursor); @@ -435,37 +436,19 @@ struct AllJoinImpl } }; -template class Impl, typename ... Args> -void dispatchKind(JoinKind kind, int null_direction_hint, Args && ... args) +template class Impl, typename ... Args> +void dispatchKind(JoinKind kind, Args && ... args) { - if (isSmall(null_direction_hint)) - { - if (Impl::enabled && kind == JoinKind::Inner) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Left) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Right) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Full) - return Impl::join(std::forward(args)...); + if (Impl::enabled && kind == JoinKind::Inner) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Left) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Right) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Full) + return Impl::join(std::forward(args)...); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); - - } - else - { - if (Impl::enabled && kind == JoinKind::Inner) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Left) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Right) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Full) - return Impl::join(std::forward(args)...); - else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); - - } } std::optional MergeJoinAlgorithm::handleAllJoinState() @@ -538,7 +521,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind) { PaddedPODArray idx_map[2]; - dispatchKind(kind, null_direction_hint, *cursors[0], *cursors[1], max_block_size, idx_map[0], idx_map[1], all_join_state); + dispatchKind(kind, *cursors[0], *cursors[1], max_block_size, idx_map[0], idx_map[1], all_join_state, null_direction_hint); assert(idx_map[0].size() == idx_map[1].size()); Chunk result; @@ -588,7 +571,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind) } -template +template struct AnyJoinImpl { constexpr static bool enabled = isInner(kind) || isLeft(kind) || isRight(kind); @@ -597,7 +580,8 @@ struct AnyJoinImpl FullMergeJoinCursor & right_cursor, PaddedPODArray & left_map, PaddedPODArray & right_map, - AnyJoinState & state) + AnyJoinState & state, + int null_direction_hint) { assert(enabled); @@ -620,7 +604,7 @@ struct AnyJoinImpl lpos = left_cursor->getRow(); rpos = right_cursor->getRow(); - cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, nullDirection(order)); + cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, null_direction_hint); if (cmp == 0) { if constexpr (isLeftOrFull(kind)) @@ -744,7 +728,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin(JoinKind kind) PaddedPODArray idx_map[2]; size_t prev_pos[] = {current_left.getRow(), current_right.getRow()}; - dispatchKind(kind, null_direction_hint, *cursors[0], *cursors[1], idx_map[0], idx_map[1], any_join_state); + dispatchKind(kind, *cursors[0], *cursors[1], idx_map[0], idx_map[1], any_join_state, null_direction_hint); assert(idx_map[0].empty() || idx_map[1].empty() || idx_map[0].size() == idx_map[1].size()); size_t num_result_rows = std::max(idx_map[0].size(), idx_map[1].size()); diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 43485321122..8af486ea34b 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -220,17 +220,6 @@ private: bool recieved_all_blocks = false; }; -/// Join method. -enum class NullOrder -{ - SMALLEST, /// null is treated as smallest - BIGGEST /// null is treated as biggest -}; - -inline constexpr bool isSmall(int null_direction) { return null_direction == 1; } - -inline constexpr int nullDirection(NullOrder order) {return order == NullOrder::SMALLEST ? 1 : -1;} - /* * This class is used to join chunks from two sorted streams. * It is used in MergeJoinTransform. From 5cf22bae6f40ab1beb258cf22a9b0627c601495d Mon Sep 17 00:00:00 2001 From: nemonlou Date: Tue, 12 Mar 2024 10:04:37 +0800 Subject: [PATCH 0217/1165] minor: fix style change --- .../Transforms/MergeJoinTransform.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 6288a850d76..37a178810cb 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -440,15 +440,15 @@ template class Impl, typename ... Args> void dispatchKind(JoinKind kind, Args && ... args) { if (Impl::enabled && kind == JoinKind::Inner) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Left) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Right) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Full) - return Impl::join(std::forward(args)...); - else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Left) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Right) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Full) + return Impl::join(std::forward(args)...); + else + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); } std::optional MergeJoinAlgorithm::handleAllJoinState() From 8d49ff7350304619386cb849851432a5ca434619 Mon Sep 17 00:00:00 2001 From: Dan Wu Date: Tue, 19 Mar 2024 03:12:32 +0000 Subject: [PATCH 0218/1165] Add tests for system.parts_columns table. --- .../03010_read_system_parts_table_test.reference | 10 ++++++++-- .../0_stateless/03010_read_system_parts_table_test.sql | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.reference b/tests/queries/0_stateless/03010_read_system_parts_table_test.reference index 6847786761e..c2dd177f245 100644 --- a/tests/queries/0_stateless/03010_read_system_parts_table_test.reference +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.reference @@ -1,4 +1,10 @@ 00000000-0000-0000-0000-000000000000 1231_1_1_0 00000000-0000-0000-0000-000000000000 6666_2_2_0 -users -users +00000000-0000-0000-0000-000000000000 1231_1_1_0 users +00000000-0000-0000-0000-000000000000 6666_2_2_0 users +00000000-0000-0000-0000-000000000000 1231_1_1_0 users uid +00000000-0000-0000-0000-000000000000 1231_1_1_0 users name +00000000-0000-0000-0000-000000000000 1231_1_1_0 users age +00000000-0000-0000-0000-000000000000 6666_2_2_0 users uid +00000000-0000-0000-0000-000000000000 6666_2_2_0 users name +00000000-0000-0000-0000-000000000000 6666_2_2_0 users age diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql index 8871822af4e..a50005b2312 100644 --- a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql @@ -6,5 +6,6 @@ INSERT INTO users VALUES (6666, 'Ksenia', 48); SELECT uuid, name from system.parts WHERE database = currentDatabase() AND table = 'users'; -SELECT table from system.parts WHERE database = currentDatabase() AND uuid = '00000000-0000-0000-0000-000000000000'; +SELECT uuid, name, table from system.parts WHERE database = currentDatabase() AND table = 'users' AND uuid = '00000000-0000-0000-0000-000000000000'; +SELECT uuid, name, table, column from system.parts_columns WHERE database = currentDatabase() AND table = 'users' AND uuid = '00000000-0000-0000-0000-000000000000'; DROP TABLE IF EXISTS users; From 8ff21d7e47c25637b87f3ab674421cfc0a2c4487 Mon Sep 17 00:00:00 2001 From: nemonlou Date: Tue, 19 Mar 2024 10:59:32 +0800 Subject: [PATCH 0219/1165] fix review comments --- src/Interpreters/FullSortingMergeJoin.h | 7 ++++++- src/Processors/QueryPlan/JoinStep.cpp | 8 +++----- src/Processors/QueryPlan/JoinStep.h | 4 +--- .../Transforms/MergeJoinTransform.cpp | 17 +++++++++++------ src/Processors/Transforms/MergeJoinTransform.h | 6 ++---- src/QueryPipeline/QueryPipelineBuilder.cpp | 3 +-- src/QueryPipeline/QueryPipelineBuilder.h | 1 - 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/Interpreters/FullSortingMergeJoin.h b/src/Interpreters/FullSortingMergeJoin.h index 7688d44f7a9..7e07c2004b6 100644 --- a/src/Interpreters/FullSortingMergeJoin.h +++ b/src/Interpreters/FullSortingMergeJoin.h @@ -21,9 +21,11 @@ namespace ErrorCodes class FullSortingMergeJoin : public IJoin { public: - explicit FullSortingMergeJoin(std::shared_ptr table_join_, const Block & right_sample_block_) + explicit FullSortingMergeJoin(std::shared_ptr table_join_, const Block & right_sample_block_, + int null_direction_ = 1) : table_join(table_join_) , right_sample_block(right_sample_block_) + , null_direction(null_direction_) { LOG_TRACE(getLogger("FullSortingMergeJoin"), "Will use full sorting merge join"); } @@ -31,6 +33,8 @@ public: std::string getName() const override { return "FullSortingMergeJoin"; } const TableJoin & getTableJoin() const override { return *table_join; } + int getNullDirection() const { return null_direction; } + bool addBlockToJoin(const Block & /* block */, bool /* check_limits */) override { throw Exception(ErrorCodes::LOGICAL_ERROR, "FullSortingMergeJoin::addBlockToJoin should not be called"); @@ -119,6 +123,7 @@ private: std::shared_ptr table_join; Block right_sample_block; Block totals; + int null_direction; }; } diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index 0c46ce7893d..1931b1eb3a1 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -44,10 +44,8 @@ JoinStep::JoinStep( JoinPtr join_, size_t max_block_size_, size_t max_streams_, - bool keep_left_read_in_order_, - int null_direction_hint_) - : join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_), - null_direction_hint(null_direction_hint_) + bool keep_left_read_in_order_) + : join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_) { updateInputStreams(DataStreams{left_stream_, right_stream_}); } @@ -60,7 +58,7 @@ QueryPipelineBuilderPtr JoinStep::updatePipeline(QueryPipelineBuilders pipelines if (join->pipelineType() == JoinPipelineType::YShaped) { auto joined_pipeline = QueryPipelineBuilder::joinPipelinesYShaped( - std::move(pipelines[0]), std::move(pipelines[1]), join, output_stream->header, max_block_size, null_direction_hint, &processors); + std::move(pipelines[0]), std::move(pipelines[1]), join, output_stream->header, max_block_size, &processors); joined_pipeline->resize(max_streams); return joined_pipeline; } diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h index 08909ce48a9..a9059a083fe 100644 --- a/src/Processors/QueryPlan/JoinStep.h +++ b/src/Processors/QueryPlan/JoinStep.h @@ -19,8 +19,7 @@ public: JoinPtr join_, size_t max_block_size_, size_t max_streams_, - bool keep_left_read_in_order_, - int null_direction_hint_ = 1); + bool keep_left_read_in_order_); String getName() const override { return "Join"; } @@ -43,7 +42,6 @@ private: size_t max_block_size; size_t max_streams; bool keep_left_read_in_order; - int null_direction_hint; }; /// Special step for the case when Join is already filled. diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 37a178810cb..b63598483ef 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -271,11 +272,9 @@ bool FullMergeJoinCursor::fullyCompleted() const MergeJoinAlgorithm::MergeJoinAlgorithm( JoinPtr table_join_, const Blocks & input_headers, - size_t max_block_size_, - int null_direction_hint_) + size_t max_block_size_) : table_join(table_join_) , max_block_size(max_block_size_) - , null_direction_hint(null_direction_hint_) , log(getLogger("MergeJoinAlgorithm")) { if (input_headers.size() != 2) @@ -305,6 +304,13 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( size_t right_idx = input_headers[1].getPositionByName(right_key); left_to_right_key_remap[left_idx] = right_idx; } + + auto smjPtr = typeid_cast(table_join.get()); + if (smjPtr) + { + null_direction_hint = smjPtr->getNullDirection(); + } + } void MergeJoinAlgorithm::logElapsed(double seconds) @@ -448,7 +454,7 @@ void dispatchKind(JoinKind kind, Args && ... args) else if (Impl::enabled && kind == JoinKind::Full) return Impl::join(std::forward(args)...); else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); } std::optional MergeJoinAlgorithm::handleAllJoinState() @@ -856,7 +862,6 @@ MergeJoinTransform::MergeJoinTransform( const Blocks & input_headers, const Block & output_header, size_t max_block_size, - int null_direction_hint_, UInt64 limit_hint_) : IMergingTransform( input_headers, @@ -865,7 +870,7 @@ MergeJoinTransform::MergeJoinTransform( limit_hint_, /* always_read_till_end_= */ false, /* empty_chunk_on_finish_= */ true, - table_join, input_headers, max_block_size, null_direction_hint_) + table_join, input_headers, max_block_size) , log(getLogger("MergeJoinTransform")) { LOG_TRACE(log, "Use MergeJoinTransform"); diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 8af486ea34b..cf9331abd59 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -227,8 +227,7 @@ private: class MergeJoinAlgorithm final : public IMergingAlgorithm { public: - explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_, - int null_direction_hint = 1); + explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_); const char * getName() const override { return "MergeJoinAlgorithm"; } void initialize(Inputs inputs) override; @@ -259,7 +258,7 @@ private: JoinPtr table_join; size_t max_block_size; - int null_direction_hint; + int null_direction_hint = 1; struct Statistic { @@ -284,7 +283,6 @@ public: const Blocks & input_headers, const Block & output_header, size_t max_block_size, - int null_direction_hint, UInt64 limit_hint = 0); String getName() const override { return "MergeJoinTransform"; } diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index e338c3ce0fa..67a8fe5dcab 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -349,7 +349,6 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesYShaped JoinPtr join, const Block & out_header, size_t max_block_size, - int null_direction_hint, Processors * collected_processors) { left->checkInitializedAndNotCompleted(); @@ -377,7 +376,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesYShaped } else { - auto joining = std::make_shared(join, inputs, out_header, max_block_size, null_direction_hint); + auto joining = std::make_shared(join, inputs, out_header, max_block_size); return mergePipelines(std::move(left), std::move(right), std::move(joining), collected_processors); } } diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index 4753f957a25..f0b2ead687e 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -137,7 +137,6 @@ public: JoinPtr table_join, const Block & out_header, size_t max_block_size, - int null_direction_hint, Processors * collected_processors = nullptr); /// Add other pipeline and execute it before current one. From a742b235a4c7cfca6f5e6ce65fd6ebd897a2364b Mon Sep 17 00:00:00 2001 From: liuneng <1398775315@qq.com> Date: Tue, 19 Mar 2024 17:12:13 +0800 Subject: [PATCH 0220/1165] add pass covert in single value to equal --- src/Analyzer/Passes/ConvertInToEqualsPass.cpp | 55 +++++++++++++++++++ src/Analyzer/Passes/ConvertInToEqualsPass.h | 24 ++++++++ src/Analyzer/QueryTreePassManager.cpp | 2 + src/Core/Settings.h | 1 + 4 files changed, 82 insertions(+) create mode 100644 src/Analyzer/Passes/ConvertInToEqualsPass.cpp create mode 100644 src/Analyzer/Passes/ConvertInToEqualsPass.h diff --git a/src/Analyzer/Passes/ConvertInToEqualsPass.cpp b/src/Analyzer/Passes/ConvertInToEqualsPass.cpp new file mode 100644 index 00000000000..cf9f1dd6ad1 --- /dev/null +++ b/src/Analyzer/Passes/ConvertInToEqualsPass.cpp @@ -0,0 +1,55 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +using FunctionEquals = FunctionComparison; + +class ConvertInToEqualsPassVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + using Base::Base; + + FunctionOverloadResolverPtr createInternalFunctionEqualOverloadResolver() + { + return std::make_unique(std::make_shared(getContext()->getSettings().decimal_check_overflow)); + } + + void enterImpl(QueryTreeNodePtr & node) + { + if (!getSettings().optimize_in_to_equal) + return; + auto * func_node = node->as(); + if (!func_node || func_node->getFunctionName() != "in" || func_node->getArguments().getNodes().size() != 2) + return ; + auto args = func_node->getArguments().getNodes(); + auto * column_node = args[0]->as(); + auto * constant_node = args[1]->as(); + if (!column_node || !constant_node) + return ; + if (constant_node->getValue().getType() == Field::Types::Which::Tuple) + return; + auto const_value = std::make_shared(constant_node->getValue()); + + auto equal_resolver = createInternalFunctionEqualOverloadResolver(); + auto equal = std::make_shared("equals"); + QueryTreeNodes arguments{column_node->clone(), const_value}; + equal->getArguments().getNodes() = std::move(arguments); + equal->resolveAsFunction(equal_resolver); + node = equal; + } +}; + +void ConvertInToEqualsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) +{ + ConvertInToEqualsPassVisitor visitor(std::move(context)); + visitor.visit(query_tree_node); +} +} diff --git a/src/Analyzer/Passes/ConvertInToEqualsPass.h b/src/Analyzer/Passes/ConvertInToEqualsPass.h new file mode 100644 index 00000000000..1f40645498a --- /dev/null +++ b/src/Analyzer/Passes/ConvertInToEqualsPass.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace DB { + +/** Optimize `in` to `equals` if possible. + * + * Example: SELECT * from test where x IN (1); + * Result: SELECT * from test where x = 1; + * + */ +class ConvertInToEqualsPass final : public IQueryTreePass { +public: + String getName() override { return "ConvertInToEqualsPass"; } + + String getDescription() override { return "Convert in to equal"; } + + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; +}; +} + + + diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 9c07884a464..284e6bc2809 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -263,6 +264,7 @@ void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze) manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); /// should before AggregateFunctionsArithmericOperationsPass manager.addPass(std::make_unique()); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index e6adb00137f..19655d6493d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -593,6 +593,7 @@ class IColumn; M(Bool, optimize_functions_to_subcolumns, false, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ M(Bool, optimize_using_constraints, false, "Use constraints for query optimization", 0) \ M(Bool, optimize_substitute_columns, false, "Use constraints for column substitution", 0) \ + M(Bool, optimize_in_to_equal, true, "Convert the constant in to equal", 0) \ M(Bool, optimize_append_index, false, "Use constraints in order to append index condition (indexHint)", 0) \ M(Bool, optimize_time_filter_with_preimage, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')", 0) \ M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \ From 75e94438e03fba799146f4ae44edbf376c48b9ce Mon Sep 17 00:00:00 2001 From: Han Fei Date: Tue, 19 Mar 2024 10:21:19 +0100 Subject: [PATCH 0221/1165] Update src/Storages/StorageReplicatedMergeTree.cpp --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 39fa65967d3..559a7074e73 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -634,7 +634,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// If mutation status is empty, than local replica may just not loaded it into memory. if (mutation_status && !mutation_status->latest_fail_reason.empty()) { - LOG_DEBUG(log, "Mutation {} is done {} or falied {} (status: '{}')", mutation_id, mutation_status->is_done, !mutation_status->latest_fail_reason.empty(), mutation_status->latest_fail_reason); + LOG_DEBUG(log, "Mutation {} is done {} or failed {} (status: '{}')", mutation_id, mutation_status->is_done, !mutation_status->latest_fail_reason.empty(), mutation_status->latest_fail_reason); break; } From 532d72a7d8b74eeb1608f103e8540a7b7deb1294 Mon Sep 17 00:00:00 2001 From: liuneng <1398775315@qq.com> Date: Tue, 19 Mar 2024 17:30:16 +0800 Subject: [PATCH 0222/1165] add test case --- ...qualsPass.cpp => ConvertInToEqualPass.cpp} | 10 ++--- ...nToEqualsPass.h => ConvertInToEqualPass.h} | 4 +- src/Analyzer/QueryTreePassManager.cpp | 4 +- tests/performance/function_in.xml | 30 +++++++++++++++ .../03013_optimize_in_to_equal.reference | 37 +++++++++++++++++++ .../03013_optimize_in_to_equal.sql | 10 +++++ 6 files changed, 86 insertions(+), 9 deletions(-) rename src/Analyzer/Passes/{ConvertInToEqualsPass.cpp => ConvertInToEqualPass.cpp} (84%) rename src/Analyzer/Passes/{ConvertInToEqualsPass.h => ConvertInToEqualPass.h} (75%) create mode 100644 tests/performance/function_in.xml create mode 100644 tests/queries/0_stateless/03013_optimize_in_to_equal.reference create mode 100644 tests/queries/0_stateless/03013_optimize_in_to_equal.sql diff --git a/src/Analyzer/Passes/ConvertInToEqualsPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp similarity index 84% rename from src/Analyzer/Passes/ConvertInToEqualsPass.cpp rename to src/Analyzer/Passes/ConvertInToEqualPass.cpp index cf9f1dd6ad1..26db3a4f4aa 100644 --- a/src/Analyzer/Passes/ConvertInToEqualsPass.cpp +++ b/src/Analyzer/Passes/ConvertInToEqualPass.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include @@ -11,10 +11,10 @@ namespace DB using FunctionEquals = FunctionComparison; -class ConvertInToEqualsPassVisitor : public InDepthQueryTreeVisitorWithContext +class ConvertInToEqualPassVisitor : public InDepthQueryTreeVisitorWithContext { public: - using Base = InDepthQueryTreeVisitorWithContext; + using Base = InDepthQueryTreeVisitorWithContext; using Base::Base; FunctionOverloadResolverPtr createInternalFunctionEqualOverloadResolver() @@ -47,9 +47,9 @@ public: } }; -void ConvertInToEqualsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) +void ConvertInToEqualPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { - ConvertInToEqualsPassVisitor visitor(std::move(context)); + ConvertInToEqualPassVisitor visitor(std::move(context)); visitor.visit(query_tree_node); } } diff --git a/src/Analyzer/Passes/ConvertInToEqualsPass.h b/src/Analyzer/Passes/ConvertInToEqualPass.h similarity index 75% rename from src/Analyzer/Passes/ConvertInToEqualsPass.h rename to src/Analyzer/Passes/ConvertInToEqualPass.h index 1f40645498a..19b83c0f65f 100644 --- a/src/Analyzer/Passes/ConvertInToEqualsPass.h +++ b/src/Analyzer/Passes/ConvertInToEqualPass.h @@ -10,9 +10,9 @@ namespace DB { * Result: SELECT * from test where x = 1; * */ -class ConvertInToEqualsPass final : public IQueryTreePass { +class ConvertInToEqualPass final : public IQueryTreePass { public: - String getName() override { return "ConvertInToEqualsPass"; } + String getName() override { return "ConvertInToEqualPass"; } String getDescription() override { return "Convert in to equal"; } diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 284e6bc2809..14eb179680c 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include #include @@ -264,7 +264,7 @@ void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze) manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); - manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); /// should before AggregateFunctionsArithmericOperationsPass manager.addPass(std::make_unique()); diff --git a/tests/performance/function_in.xml b/tests/performance/function_in.xml new file mode 100644 index 00000000000..16e6bb91469 --- /dev/null +++ b/tests/performance/function_in.xml @@ -0,0 +1,30 @@ + + + 8 + 1 + + + + CREATE TABLE t_nullable + ( + key_string1 Nullable(String), + key_string2 Nullable(String), + key_string3 Nullable(String), + key_int64_1 Nullable(Int64), + key_int64_2 Nullable(Int64), + key_int64_3 Nullable(Int64), + key_int64_4 Nullable(Int64), + key_int64_5 Nullable(Int64), + m1 Int64, + m2 Int64 + ) + ENGINE = Memory + + insert into t_nullable select ['aaaaaa','bbaaaa','ccaaaa','ddaaaa'][number % 101 + 1], ['aa','bb','cc','dd'][number % 100 + 1], ['aa','bb','cc','dd'][number % 102 + 1], number%10+1, number%10+2, number%10+3, number%10+4,number%10+5, number%6000+1, number%5000+2 from numbers_mt(30000000) + select * from t_nullable where key_string1 in ('aaaaaa') format Null + select * from t_nullable where key_string1 in ('aaaaaa') format Null SETTINGS allow_experimental_analyzer=1 + select * from t_nullable where key_string2 in ('3') format Null + select * from t_nullable where key_string2 in ('3') format Null SETTINGS allow_experimental_analyzer=1 + drop table if exists t_nullable + + \ No newline at end of file diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.reference b/tests/queries/0_stateless/03013_optimize_in_to_equal.reference new file mode 100644 index 00000000000..f55a7b76eaf --- /dev/null +++ b/tests/queries/0_stateless/03013_optimize_in_to_equal.reference @@ -0,0 +1,37 @@ +a 1 +------------------- +QUERY id: 0 + PROJECTION COLUMNS + x String + y Int32 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: x, result_type: String, source_id: 3 + COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.test + WHERE + FUNCTION id: 5, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + COLUMN id: 7, column_name: x, result_type: String, source_id: 3 + CONSTANT id: 8, constant_value: \'a\', constant_value_type: String + SETTINGS allow_experimental_analyzer=1 +------------------- +QUERY id: 0 + PROJECTION COLUMNS + x String + y Int32 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: x, result_type: String, source_id: 3 + COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.test + WHERE + FUNCTION id: 5, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + COLUMN id: 7, column_name: x, result_type: String, source_id: 3 + CONSTANT id: 8, constant_value: Tuple_(\'a\', \'b\'), constant_value_type: Tuple(String, String) + SETTINGS allow_experimental_analyzer=1 diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.sql b/tests/queries/0_stateless/03013_optimize_in_to_equal.sql new file mode 100644 index 00000000000..9d80825ceb4 --- /dev/null +++ b/tests/queries/0_stateless/03013_optimize_in_to_equal.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS test; +CREATE TABLE test (x String, y Int32) ENGINE = MergeTree() ORDER BY x; + +INSERT INTO test VALUES ('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5); + +select * from test where x in ('a') SETTINGS allow_experimental_analyzer = 1; +select '-------------------'; +explain query tree select * from test where x in ('a') SETTINGS allow_experimental_analyzer = 1; +select '-------------------'; +explain query tree select * from test where x in ('a','b') SETTINGS allow_experimental_analyzer = 1; \ No newline at end of file From 24a59546c7247c596627f708754d7fc7e6a08093 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 19 Mar 2024 09:42:40 +0000 Subject: [PATCH 0223/1165] ASdd documentation --- docs/en/sql-reference/table-functions/generateSeries.md | 8 ++++++++ docs/en/sql-reference/table-functions/generate_series.md | 5 +---- ...4_all_new_table_functions_must_be_documented.reference | 1 + .../0_stateless/02992_all_columns_should_have_comment.sql | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 docs/en/sql-reference/table-functions/generateSeries.md diff --git a/docs/en/sql-reference/table-functions/generateSeries.md b/docs/en/sql-reference/table-functions/generateSeries.md new file mode 100644 index 00000000000..9dd2e09b6a7 --- /dev/null +++ b/docs/en/sql-reference/table-functions/generateSeries.md @@ -0,0 +1,8 @@ +--- +slug: /en/sql-reference/table-functions/generateSeries +sidebar_position: 147 +sidebar_label: generateSeries +--- + +### AliasTo +[generate_series](generate_series.md) \ No newline at end of file diff --git a/docs/en/sql-reference/table-functions/generate_series.md b/docs/en/sql-reference/table-functions/generate_series.md index 9370cea8580..c5d29369627 100644 --- a/docs/en/sql-reference/table-functions/generate_series.md +++ b/docs/en/sql-reference/table-functions/generate_series.md @@ -22,7 +22,4 @@ And the following queries return tables with the same content but different colu ``` sql SELECT * FROM numbers(10, 11) WHERE number % 3 == (10 % 3); SELECT * FROM generate_series(10, 20, 3) ; -``` - -### Alias -generateSeries \ No newline at end of file +``` \ No newline at end of file diff --git a/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference b/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference index 1e4f21a6722..57ffeca9d43 100644 --- a/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference @@ -4,6 +4,7 @@ dictionary executable file generateRandom +generateSeries generate_series input jdbc diff --git a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql index b90faeac359..e233f08cc79 100644 --- a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql +++ b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql @@ -1,4 +1,4 @@ SYSTEM FLUSH LOGS; SELECT 'Column ' || name || ' from table ' || concat(database, '.', table) || ' should have a comment' FROM system.columns -WHERE (database = 'system') AND (comment = '') AND (table NOT ILIKE '%_log_%') AND (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series')) AND (default_kind != 'ALIAS'); +WHERE (database = 'system') AND (comment = '') AND (table NOT ILIKE '%_log_%') AND (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series', 'generateSeries')) AND (default_kind != 'ALIAS'); From dba72bf9c5817a96ea083cdb3edf5e3d0ded08df Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Mar 2024 10:02:35 +0000 Subject: [PATCH 0224/1165] Fix --- src/Client/ClientBase.cpp | 8 -------- src/Client/ClientBase.h | 1 - src/Core/SettingsChangesHistory.h | 1 + 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index de995ffe408..5ec18c41f05 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -68,9 +68,6 @@ #include #include -#include -#include - #include #include #include @@ -562,11 +559,6 @@ try pager_cmd = ShellCommand::execute(config); out_buf = &pager_cmd->in; } - /// We can use special buffer for query output for internal queries. - else if (output_format_buffer) - { - out_buf = output_format_buffer.get(); - } else { out_buf = &std_out; diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 74d065bf342..d01a69f14dc 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -248,7 +248,6 @@ protected: /// The user can specify to redirect query output to a file. std::unique_ptr out_file_buf; std::shared_ptr output_format; - std::unique_ptr output_format_buffer; /// The user could specify special file for server logs (stderr by default) std::unique_ptr out_logs_buf; diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 072b9803682..c8d57626136 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -87,6 +87,7 @@ static std::map sett { {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, + {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"}, {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"}, {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"}, {"page_cache_inject_eviction", false, false, "Added userspace page cache"}, From 761c612295c8f09ff12fc697e482069ac7bd61a2 Mon Sep 17 00:00:00 2001 From: liuneng <1398775315@qq.com> Date: Tue, 19 Mar 2024 18:29:07 +0800 Subject: [PATCH 0225/1165] fix style --- src/Analyzer/Passes/ConvertInToEqualPass.h | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.h b/src/Analyzer/Passes/ConvertInToEqualPass.h index 19b83c0f65f..7de75e94f04 100644 --- a/src/Analyzer/Passes/ConvertInToEqualPass.h +++ b/src/Analyzer/Passes/ConvertInToEqualPass.h @@ -2,15 +2,15 @@ #include -namespace DB { - +namespace DB +{ /** Optimize `in` to `equals` if possible. * * Example: SELECT * from test where x IN (1); * Result: SELECT * from test where x = 1; - * */ -class ConvertInToEqualPass final : public IQueryTreePass { +class ConvertInToEqualPass final : public IQueryTreePass +{ public: String getName() override { return "ConvertInToEqualPass"; } @@ -19,6 +19,3 @@ public: void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; }; } - - - From 87e2c09febe17ce45e29d7d02397193f23b250cc Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 19 Mar 2024 10:31:05 +0000 Subject: [PATCH 0226/1165] Correct typo --- docs/en/sql-reference/table-functions/generateSeries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/table-functions/generateSeries.md b/docs/en/sql-reference/table-functions/generateSeries.md index 9dd2e09b6a7..addf539a959 100644 --- a/docs/en/sql-reference/table-functions/generateSeries.md +++ b/docs/en/sql-reference/table-functions/generateSeries.md @@ -4,5 +4,5 @@ sidebar_position: 147 sidebar_label: generateSeries --- -### AliasTo +### Alias To [generate_series](generate_series.md) \ No newline at end of file From 462445c2c201a5f1f9b773ecd6c3819699ecb3ba Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Mar 2024 10:41:23 +0000 Subject: [PATCH 0227/1165] Fix logical error in group_by_use_nulls + analyzer + materialize/constant --- src/Analyzer/ConstantNode.h | 6 ++++++ .../Passes/OptimizeGroupByFunctionKeysPass.cpp | 4 +++- .../Passes/OptimizeGroupByInjectiveFunctionsPass.cpp | 8 ++++++++ src/Functions/materialize.h | 2 ++ ..._use_nulls_with_materialize_and_analyzer.reference | 10 ++++++++++ ...oup_by_use_nulls_with_materialize_and_analyzer.sql | 11 +++++++++++ 6 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.reference create mode 100644 tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql diff --git a/src/Analyzer/ConstantNode.h b/src/Analyzer/ConstantNode.h index b065853e315..98a8eb78277 100644 --- a/src/Analyzer/ConstantNode.h +++ b/src/Analyzer/ConstantNode.h @@ -4,6 +4,7 @@ #include #include +#include namespace DB { @@ -86,6 +87,11 @@ public: mask_id = id; } + void convertToNullable() override + { + constant_value = std::make_shared(constant_value->getValue(), makeNullableSafe(constant_value->getType())); + } + void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp index d881af3a51b..0cd9a5a0cd2 100644 --- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp @@ -73,12 +73,14 @@ private: candidates.push_back({ *it, is_deterministic }); /// Using DFS we traverse function tree and try to find if it uses other keys as function arguments. + bool found_at_least_one_usage = false; while (!candidates.empty()) { auto [candidate, parents_are_only_deterministic] = candidates.back(); candidates.pop_back(); bool found = group_by_keys.contains(candidate); + found_at_least_one_usage |= found; switch (candidate->getNodeType()) { @@ -111,7 +113,7 @@ private: } } - return true; + return found_at_least_one_usage; } static void optimizeGroupingSet(QueryTreeNodes & grouping_set) diff --git a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp index ad649834fb0..11d79c7b2d9 100644 --- a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp @@ -97,12 +97,20 @@ private: if (can_be_eliminated) { + bool all_arguments_are_constants = true; for (auto const & argument : function_node->getArguments()) { // We can skip constants here because aggregation key is already not a constant. if (argument->getNodeType() != QueryTreeNodeType::CONSTANT) + { + all_arguments_are_constants = false; nodes_to_process.push(argument); + } } + + /// We cannot optimize function if it has only constant arguments (for example, materialize(const_value)). + if (all_arguments_are_constants) + new_group_by_keys.push_back(node_to_process); } else new_group_by_keys.push_back(node_to_process); diff --git a/src/Functions/materialize.h b/src/Functions/materialize.h index 73bfdec48ab..41994509745 100644 --- a/src/Functions/materialize.h +++ b/src/Functions/materialize.h @@ -36,6 +36,8 @@ public: bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } + bool isSuitableForConstantFolding() const override { return false; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } size_t getNumberOfArguments() const override diff --git a/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.reference b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.reference new file mode 100644 index 00000000000..90deeca2e36 --- /dev/null +++ b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.reference @@ -0,0 +1,10 @@ +6 +6 +3 +\N +0 +\N +\N +0 +\N +0 diff --git a/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql new file mode 100644 index 00000000000..2f180d8aedd --- /dev/null +++ b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql @@ -0,0 +1,11 @@ +set allow_experimental_analyzer = 1; +set group_by_use_nulls = 1; +set optimize_group_by_function_keys = 1; +set optimize_injective_functions_in_group_by = 1; + +SELECT 3 + 3 from numbers(10) GROUP BY GROUPING SETS (('str'), (3 + 3)); +SELECT materialize(3) from numbers(10) GROUP BY GROUPING SETS (('str'), (materialize(3))); +SELECT ignore(3) from numbers(10) GROUP BY GROUPING SETS (('str'), (ignore(3))); +SELECT materialize(ignore(3)) from numbers(10) GROUP BY GROUPING SETS (('str'), (materialize(ignore(3)))); +SELECT ignore(materialize(3)) from numbers(10) GROUP BY GROUPING SETS (('str'), (ignore(materialize(3)))); + From eb75926e50d03ddba7327c3377b11ead3e221c80 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 21 Feb 2024 14:33:01 +0100 Subject: [PATCH 0228/1165] Parallel flush of pending INSERT blocks of Distributed engine Parallelism will work only if you have multi disk policy for table (like everything in Distributed engine right now). This will work for DETACH/server shutdown and SYSTEM FLUSH DISTRIBUTED Signed-off-by: Azat Khuzhin --- src/Storages/StorageDistributed.cpp | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 3b766ac8d26..6554af1f7e6 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1734,16 +1734,27 @@ void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context) directory_queues.push_back(node.second.directory_queue); } - bool need_flush = getDistributedSettingsRef().flush_on_detach; - if (!need_flush) + if (getDistributedSettingsRef().flush_on_detach) + { + LOG_INFO(log, "Flushing pending INSERT blocks"); + + Stopwatch watch; + ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, directory_queues.size()); + for (const auto & node : directory_queues) + { + pool.scheduleOrThrowOnError([node_to_flush = node]() + { + node_to_flush->flushAllData(); + }); + } + pool.wait(); + LOG_INFO(log, "Pending INSERT blocks flushed, took {} ms.", watch.elapsedMilliseconds()); + } + else + { LOG_INFO(log, "Skip flushing data (due to flush_on_detach=0)"); - /// TODO: Maybe it should be executed in parallel - for (auto & node : directory_queues) - { - if (need_flush) - node->flushAllData(); - else + for (auto & node : directory_queues) node->shutdownWithoutFlush(); } } From 929dc6fa12ced13309fea73229a0c5213cdc0f30 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 22 Feb 2024 15:41:19 +0100 Subject: [PATCH 0229/1165] Attach to query for threads in Distributed engine background ops Signed-off-by: Azat Khuzhin --- src/Storages/StorageDistributed.cpp | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 6554af1f7e6..e25e3425359 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1287,10 +1287,19 @@ void StorageDistributed::initializeFromDisk() /// Make initialization for large number of disks parallel. ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, disks.size()); + ThreadGroupPtr thread_group = CurrentThread::getGroup(); for (const DiskPtr & disk : disks) { pool.scheduleOrThrowOnError([&]() { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); + if (thread_group) + CurrentThread::attachToGroup(thread_group); + setThreadName("DistInit"); + initializeDirectoryQueuesForDisk(disk); }); } @@ -1302,6 +1311,14 @@ void StorageDistributed::initializeFromDisk() { pool.scheduleOrThrowOnError([&, i]() { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); + if (thread_group) + CurrentThread::attachToGroup(thread_group); + setThreadName("DistInit"); + last_increment[i] = getMaximumFileNumber(paths[i]); }); } @@ -1739,11 +1756,20 @@ void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context) LOG_INFO(log, "Flushing pending INSERT blocks"); Stopwatch watch; + ThreadGroupPtr thread_group = CurrentThread::getGroup(); ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, directory_queues.size()); for (const auto & node : directory_queues) { - pool.scheduleOrThrowOnError([node_to_flush = node]() + pool.scheduleOrThrowOnError([node_to_flush = node, &thread_group]() { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); + if (thread_group) + CurrentThread::attachToGroup(thread_group); + setThreadName("DistFlush"); + node_to_flush->flushAllData(); }); } From 667aade4e31aedebf6429a6119dd9a5991fbe258 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 19 Mar 2024 12:12:35 +0100 Subject: [PATCH 0230/1165] Fix build --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 559a7074e73..ddd9bf9fbae 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -641,7 +641,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Replica can become inactive, so wait with timeout, if nothing happened -> recheck it if (!wait_event->tryWait(1000)) { - LOG_TRACE(log, "Failed to wait for mutation '{}', will recheck", mutation_id) + LOG_TRACE(log, "Failed to wait for mutation '{}', will recheck", mutation_id); } } From 8a28fe3bac0aa413a8eb4a451f7902a248d7e17d Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Mar 2024 11:44:05 +0000 Subject: [PATCH 0231/1165] Fix tests --- .../02516_projections_and_context.reference | 2 -- .../0_stateless/02516_projections_and_context.sql | 4 ++-- ...y_use_nulls_with_materialize_and_analyzer.reference | 2 +- ...roup_by_use_nulls_with_materialize_and_analyzer.sql | 10 +++++----- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/02516_projections_and_context.reference b/tests/queries/0_stateless/02516_projections_and_context.reference index 6ed281c757a..e69de29bb2d 100644 --- a/tests/queries/0_stateless/02516_projections_and_context.reference +++ b/tests/queries/0_stateless/02516_projections_and_context.reference @@ -1,2 +0,0 @@ -1 -1 diff --git a/tests/queries/0_stateless/02516_projections_and_context.sql b/tests/queries/0_stateless/02516_projections_and_context.sql index 2b659eafabc..334544eb4fa 100644 --- a/tests/queries/0_stateless/02516_projections_and_context.sql +++ b/tests/queries/0_stateless/02516_projections_and_context.sql @@ -5,6 +5,6 @@ set allow_experimental_analyzer = 0; SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL)); -- { serverError BAD_ARGUMENTS } SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a')); -- { serverError BAD_ARGUMENTS } set allow_experimental_analyzer = 1; -SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL)); -SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a')); +SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL)); -- { serverError BAD_ARGUMENTS } +SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a')); -- { serverError BAD_ARGUMENTS } DROP TABLE test1__fuzz_37; diff --git a/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.reference b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.reference index 90deeca2e36..2c9c098cea0 100644 --- a/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.reference +++ b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.reference @@ -4,7 +4,7 @@ \N 0 \N -\N 0 \N 0 +\N diff --git a/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql index 2f180d8aedd..b15593b2abd 100644 --- a/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql +++ b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql @@ -3,9 +3,9 @@ set group_by_use_nulls = 1; set optimize_group_by_function_keys = 1; set optimize_injective_functions_in_group_by = 1; -SELECT 3 + 3 from numbers(10) GROUP BY GROUPING SETS (('str'), (3 + 3)); -SELECT materialize(3) from numbers(10) GROUP BY GROUPING SETS (('str'), (materialize(3))); -SELECT ignore(3) from numbers(10) GROUP BY GROUPING SETS (('str'), (ignore(3))); -SELECT materialize(ignore(3)) from numbers(10) GROUP BY GROUPING SETS (('str'), (materialize(ignore(3)))); -SELECT ignore(materialize(3)) from numbers(10) GROUP BY GROUPING SETS (('str'), (ignore(materialize(3)))); +SELECT 3 + 3 from numbers(10) GROUP BY GROUPING SETS (('str'), (3 + 3)) order by all; +SELECT materialize(3) from numbers(10) GROUP BY GROUPING SETS (('str'), (materialize(3))) order by all; +SELECT ignore(3) from numbers(10) GROUP BY GROUPING SETS (('str'), (ignore(3))) order by all; +SELECT materialize(ignore(3)) from numbers(10) GROUP BY GROUPING SETS (('str'), (materialize(ignore(3)))) order by all; +SELECT ignore(materialize(3)) from numbers(10) GROUP BY GROUPING SETS (('str'), (ignore(materialize(3)))) order by all; From f0b955888b304b952d4ab32941712dc492860b4e Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 18 Mar 2024 18:46:17 +0000 Subject: [PATCH 0232/1165] Fix tryInsert for ColumnUInt8 and Bool --- src/Columns/ColumnVector.cpp | 22 ++++++++++++++++++++++ src/Columns/ColumnVector.h | 10 ++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 015a114105d..4e3b9963107 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -460,6 +460,28 @@ Float32 ColumnVector::getFloat32(size_t n [[maybe_unused]]) const throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Float32", TypeName); } +template +bool ColumnVector::tryInsert(const DB::Field & x) +{ + NearestFieldType value; + if (!x.tryGet>(value)) + { + if constexpr (std::is_same_v) + { + /// It's also possible to insert boolean values into UInt8 column. + bool boolean_value; + if (x.tryGet(boolean_value)) + { + data.push_back(static_cast(boolean_value)); + return true; + } + } + return false; + } + data.push_back(static_cast(value)); + return true; +} + template void ColumnVector::insertRangeFrom(const IColumn & src, size_t start, size_t length) { diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index b03d476a8f1..39ee1d931bd 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -224,14 +224,8 @@ public: data.push_back(static_cast(x.get())); } - bool tryInsert(const DB::Field & x) override - { - NearestFieldType value; - if (!x.tryGet>(value)) - return false; - data.push_back(static_cast(value)); - return true; - } + bool tryInsert(const DB::Field & x) override; + void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override; From 9b14d17d7b207ccaa83a7e51b90ab9be79782d69 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 19 Mar 2024 12:58:36 +0100 Subject: [PATCH 0233/1165] Fxi --- src/Storages/StorageReplicatedMergeTree.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ddd9bf9fbae..30646e0138c 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -673,7 +673,8 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// they will happen on each replica, so we can check only in-memory info. auto mutation_status = queue.getIncompleteMutationsStatus(mutation_id, &mutation_ids); checkMutationStatus(mutation_status, mutation_ids); - chassert(mutation_status.is_done); + /// Mutation should be done or we should throw exception + chassert(mutation_status->is_done); } if (!inactive_replicas.empty()) From 8b1df937a567cec4acc6702288d979d4489bb83f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 19 Mar 2024 07:09:28 -0400 Subject: [PATCH 0234/1165] Use scheduleFromThreadPool for parallel processing in distributed storage Signed-off-by: Azat Khuzhin --- src/Storages/StorageDistributed.cpp | 63 ++++++++++++++--------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index e25e3425359..7819aa7683a 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -23,6 +23,7 @@ #include +#include #include #include #include @@ -282,6 +283,17 @@ size_t getClusterQueriedNodes(const Settings & settings, const ClusterPtr & clus return (num_remote_shards + num_local_shards) * settings.max_parallel_replicas; } +template +void waitFutures(F & futures) +{ + for (auto & future : futures) + future.wait(); + /// Make sure there is no exception. + for (auto & future : futures) + future.get(); + futures.clear(); +} + } /// For destruction of std::unique_ptr of type that is incomplete in class definition. @@ -1286,42 +1298,30 @@ void StorageDistributed::initializeFromDisk() /// Make initialization for large number of disks parallel. ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, disks.size()); + std::vector> futures; - ThreadGroupPtr thread_group = CurrentThread::getGroup(); for (const DiskPtr & disk : disks) { - pool.scheduleOrThrowOnError([&]() + auto future = scheduleFromThreadPool([this, disk_to_init = disk] { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachFromGroupIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachToGroup(thread_group); - setThreadName("DistInit"); - - initializeDirectoryQueuesForDisk(disk); - }); + initializeDirectoryQueuesForDisk(disk_to_init); + }, pool, "DistInit"); + futures.push_back(std::move(future)); } + waitFutures(futures); pool.wait(); const auto & paths = getDataPaths(); std::vector last_increment(paths.size()); for (size_t i = 0; i < paths.size(); ++i) { - pool.scheduleOrThrowOnError([&, i]() + auto future = scheduleFromThreadPool([&paths, &last_increment, i] { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachFromGroupIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachToGroup(thread_group); - setThreadName("DistInit"); - last_increment[i] = getMaximumFileNumber(paths[i]); - }); + }, pool, "DistInit"); + futures.push_back(std::move(future)); } + waitFutures(futures); pool.wait(); for (const auto inc : last_increment) @@ -1756,24 +1756,21 @@ void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context) LOG_INFO(log, "Flushing pending INSERT blocks"); Stopwatch watch; - ThreadGroupPtr thread_group = CurrentThread::getGroup(); ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, directory_queues.size()); + std::vector> futures; + for (const auto & node : directory_queues) { - pool.scheduleOrThrowOnError([node_to_flush = node, &thread_group]() + auto future = scheduleFromThreadPool([node_to_flush = node] { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachFromGroupIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachToGroup(thread_group); - setThreadName("DistFlush"); - node_to_flush->flushAllData(); - }); + }, pool, "DistFlush"); + futures.push_back(std::move(future)); } + + waitFutures(futures); pool.wait(); + LOG_INFO(log, "Pending INSERT blocks flushed, took {} ms.", watch.elapsedMilliseconds()); } else From 44b985a2bc9afd94fd206bb8699eabb69fae4e3f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:26:32 +0100 Subject: [PATCH 0235/1165] Update install.md From 8494e7389d21307781b91a030e22f05dac6deeb7 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 19 Mar 2024 13:47:20 +0100 Subject: [PATCH 0236/1165] Update setting names --- src/Common/BufferAllocationPolicy.cpp | 14 +++++++------- src/Common/BufferAllocationPolicy.h | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Common/BufferAllocationPolicy.cpp b/src/Common/BufferAllocationPolicy.cpp index 2b00d4f84d0..14d009008b9 100644 --- a/src/Common/BufferAllocationPolicy.cpp +++ b/src/Common/BufferAllocationPolicy.cpp @@ -12,7 +12,7 @@ class FixedSizeBufferAllocationPolicy : public IBufferAllocationPolicy public: explicit FixedSizeBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_) - : buffer_size(settings_.strict_upload_part_size) + : buffer_size(settings_.strict_size) { chassert(buffer_size > 0); } @@ -46,11 +46,11 @@ class ExpBufferAllocationPolicy : public DB::IBufferAllocationPolicy public: explicit ExpBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_) - : first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size)) - , second_size(settings_.min_upload_part_size) - , multiply_factor(settings_.upload_part_size_multiply_factor) - , multiply_threshold(settings_.upload_part_size_multiply_parts_count_threshold) - , max_size(settings_.max_upload_part_size) + : first_size(std::max(settings_.max_single_size, settings_.min_size)) + , second_size(settings_.min_size) + , multiply_factor(settings_.multiply_factor) + , multiply_threshold(settings_.multiply_parts_count_threshold) + , max_size(settings_.max_size) { chassert(first_size > 0); chassert(second_size > 0); @@ -93,7 +93,7 @@ IBufferAllocationPolicy::~IBufferAllocationPolicy() = default; IBufferAllocationPolicy::IBufferAllocationPolicyPtr IBufferAllocationPolicy::create(IBufferAllocationPolicy::Settings settings_) { - if (settings_.strict_upload_part_size > 0) + if (settings_.strict_size > 0) return std::make_unique(settings_); else return std::make_unique(settings_); diff --git a/src/Common/BufferAllocationPolicy.h b/src/Common/BufferAllocationPolicy.h index 3ada9e52fdf..dd04139e694 100644 --- a/src/Common/BufferAllocationPolicy.h +++ b/src/Common/BufferAllocationPolicy.h @@ -15,12 +15,12 @@ public: struct Settings { - size_t strict_upload_part_size = 0; - size_t min_upload_part_size = 16 * 1024 * 1024; - size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; - size_t upload_part_size_multiply_factor = 2; - size_t upload_part_size_multiply_parts_count_threshold = 500; - size_t max_single_part_upload_size = 32 * 1024 * 1024; + size_t strict_size = 0; + size_t min_size = 16 * 1024 * 1024; + size_t max_size = 5ULL * 1024 * 1024 * 1024; + size_t multiply_factor = 2; + size_t multiply_parts_count_threshold = 500; + size_t max_single_size = 32 * 1024 * 1024; }; From c16922493a614779d989701b9bed4195a503d6e9 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 19 Mar 2024 14:54:06 +0100 Subject: [PATCH 0237/1165] Removed detachPart & reallocateBuffer functions --- .../IO/WriteBufferFromAzureBlobStorage.cpp | 58 +++++-------------- .../IO/WriteBufferFromAzureBlobStorage.h | 7 +-- 2 files changed, 16 insertions(+), 49 deletions(-) diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index 41d7187afda..50763f7f189 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -18,12 +18,6 @@ namespace ProfileEvents namespace DB { -struct WriteBufferFromAzureBlobStorage::PartData -{ - Memory<> memory; - size_t data_size = 0; -}; - WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( std::shared_ptr blob_container_client_, const String & blob_path_, @@ -108,53 +102,31 @@ void WriteBufferFromAzureBlobStorage::finalizeImpl() void WriteBufferFromAzureBlobStorage::nextImpl() { task_tracker->waitIfAny(); - - reallocateBuffer(); - detachBuffer(); - - while (!detached_part_data.empty()) - { - writePart(std::move(detached_part_data.front())); - detached_part_data.pop_front(); - } - + writePart(); allocateBuffer(); } void WriteBufferFromAzureBlobStorage::allocateBuffer() { buffer_allocation_policy->nextBuffer(); - memory = Memory(buffer_allocation_policy->getBufferSize()); + auto size = buffer_allocation_policy->getBufferSize(); + + if (buffer_allocation_policy->getBufferNumber() == 1) + { + size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), size); + } + + memory = Memory(size); WriteBuffer::set(memory.data(), memory.size()); } - -void WriteBufferFromAzureBlobStorage::reallocateBuffer() -{ - if (available() > 0) - return; - - if (memory.size() == buffer_allocation_policy->getBufferSize()) - return; - - memory.resize(buffer_allocation_policy->getBufferSize()); - - WriteBuffer::set(memory.data(), memory.size()); - - chassert(offset() == 0); -} - -void WriteBufferFromAzureBlobStorage::detachBuffer() +void WriteBufferFromAzureBlobStorage::writePart() { size_t data_size = size_t(position() - memory.data()); - auto buf = std::move(memory); + auto data = std::move(memory); WriteBuffer::set(nullptr, 0); - detached_part_data.push_back({std::move(buf), data_size}); -} -void WriteBufferFromAzureBlobStorage::writePart(WriteBufferFromAzureBlobStorage::PartData && data) -{ - if (data.data_size == 0) + if (data_size == 0) return; auto upload_worker = [&] () @@ -162,11 +134,11 @@ void WriteBufferFromAzureBlobStorage::writePart(WriteBufferFromAzureBlobStorage: auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); const std::string & block_id = block_ids.emplace_back(getRandomASCIIString(64)); - Azure::Core::IO::MemoryBodyStream memory_stream(reinterpret_cast(data.memory.data()), data.data_size); - execWithRetry([&](){ block_blob_client.StageBlock(block_id, memory_stream); }, max_unexpected_write_error_retries, data.data_size); + Azure::Core::IO::MemoryBodyStream memory_stream(reinterpret_cast(data.data()), data_size); + execWithRetry([&](){ block_blob_client.StageBlock(block_id, memory_stream); }, max_unexpected_write_error_retries, data_size); if (write_settings.remote_throttler) - write_settings.remote_throttler->add(data.data_size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); + write_settings.remote_throttler->add(data_size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); }; task_tracker->add(std::move(upload_worker)); diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index f047dd53724..a210a75030a 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -48,12 +48,8 @@ public: private: struct PartData; - void writePart(WriteBufferFromAzureBlobStorage::PartData && data); - void detachBuffer(); + void writePart(); void allocateBuffer(); - void allocateFirstBuffer(); - void reallocateFirstBuffer(); - void reallocateBuffer(); void finalizeImpl() override; void execWithRetry(std::function func, size_t num_tries, size_t cost = 0); @@ -81,7 +77,6 @@ private: bool first_buffer=true; std::unique_ptr task_tracker; - std::deque detached_part_data; }; } From 6a29268d5a22ba19df80ead6f34cddf942f7bf41 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Mar 2024 12:53:28 +0100 Subject: [PATCH 0238/1165] Fix --- src/Interpreters/Cache/EvictionCandidates.cpp | 1 + src/Interpreters/Cache/FileCache.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 7f77f14ee76..997b770f953 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -37,6 +37,7 @@ void EvictionCandidates::add(const FileSegmentMetadataPtr & candidate, LockedKey it->second.key_metadata = locked_key.getKeyMetadata(); it->second.candidates.push_back(candidate); + chassert(!candidate->isEvicting(lock)); candidate->setEvicting(true, &locked_key, &lock); ++candidates_size; } diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 7d6df156cc0..e480fb559d4 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -853,11 +853,11 @@ bool FileCache::tryReserve( { chassert(reached_size_limit || reached_elements_limit); - std::unique_ptr hold_space; - if (!reached_size_limit) - hold_space = std::make_unique(size, 0, queue_iterator, *main_priority, cache_lock); - else if (!reached_elements_limit) - hold_space = std::make_unique(0, 1, queue_iterator, *main_priority, cache_lock); + size_t hold_size = reached_size_limit + ? size > reserve_stat.stat.releasable_size ? size - reserve_stat.stat.releasable_size : 0 + : size; + size_t hold_elements = reached_elements_limit ? 0 : 1; + auto hold_space = std::make_unique(hold_size, hold_elements, queue_iterator, *main_priority, cache_lock); cache_lock.unlock(); try From 939554ec0a62aa5a1b0395a72b95a17be8578d1b Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Sat, 9 Mar 2024 07:52:23 +0000 Subject: [PATCH 0239/1165] contrib: update cppkafka to v0.4.1@9c5ea0e3 --- contrib/cppkafka | 2 +- src/Storages/Kafka/KafkaConsumer.cpp | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/contrib/cppkafka b/contrib/cppkafka index 5a119f689f8..9c5ea0e3324 160000 --- a/contrib/cppkafka +++ b/contrib/cppkafka @@ -1 +1 @@ -Subproject commit 5a119f689f8a4d90d10a9635e7ee2bee5c127de1 +Subproject commit 9c5ea0e332486961e612deacc6e3f0c1874c688d diff --git a/src/Storages/Kafka/KafkaConsumer.cpp b/src/Storages/Kafka/KafkaConsumer.cpp index 47167e19a38..7075dcb71ca 100644 --- a/src/Storages/Kafka/KafkaConsumer.cpp +++ b/src/Storages/Kafka/KafkaConsumer.cpp @@ -599,24 +599,14 @@ void KafkaConsumer::setExceptionInfo(const std::string & text, bool with_stacktr exceptions_buffer.push_back({enriched_text, static_cast(Poco::Timestamp().epochTime())}); } -/* - * Needed until - * https://github.com/mfontanini/cppkafka/pull/309 - * is merged, - * because consumer->get_member_id() contains a leak - */ std::string KafkaConsumer::getMemberId() const { if (!consumer) return ""; - char * memberid_ptr = rd_kafka_memberid(consumer->get_handle()); - std::string memberid_string = memberid_ptr; - rd_kafka_mem_free(nullptr, memberid_ptr); - return memberid_string; + return consumer->get_member_id(); } - KafkaConsumer::Stat KafkaConsumer::getStat() const { KafkaConsumer::Stat::Assignments assignments; From 4e2e5a015b162d63dd192dc1cdb0cb622a524cf1 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:29:50 +0100 Subject: [PATCH 0240/1165] Fixes --- src/Interpreters/InterpreterCreateQuery.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index fb421ee0a5e..38918918c92 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1087,6 +1087,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) // If this is a stub ATTACH query, read the query definition from the database if (create.attach && !create.storage && !create.columns_list) { + // In case of an ON CLUSTER query, the database may not be present on the initiator node auto database = DatabaseCatalog::instance().tryGetDatabase(database_name); if (database && database->shouldReplicateQuery(getContext(), query_ptr)) { @@ -1253,6 +1254,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) DatabasePtr database; bool need_add_to_database = !create.temporary; + // In case of an ON CLUSTER query, the database may not be present on the initiator node if (need_add_to_database) database = DatabaseCatalog::instance().tryGetDatabase(database_name); @@ -1273,7 +1275,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) "CREATE AS SELECT is not supported with Replicated databases. Use separate CREATE and INSERT queries"); } - if (need_add_to_database && database && database->shouldReplicateQuery(getContext(), query_ptr)) + if (database && database->shouldReplicateQuery(getContext(), query_ptr)) { chassert(!ddl_guard); auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); From 56e48c5d64c6e9ab3441908c04014f3e3e87de74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 16:56:46 +0100 Subject: [PATCH 0241/1165] Increase compiler memory for riscv --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c4e16eace2..d345705ae53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,8 +61,8 @@ if (ENABLE_CHECK_HEAVY_BUILDS) # set CPU time limit to 1000 seconds set (RLIMIT_CPU 1000) - # -fsanitize=memory and address are too heavy - if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE) + # Sanitizers are too heavy. Also RISCV has some extra memory requirements + if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE OR ARCH_RISCV64) set (RLIMIT_DATA 10000000000) # 10G endif() From de855ca917798adea684d25bfb8d287303c30d70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 17:04:29 +0100 Subject: [PATCH 0242/1165] Reduce header dependencies --- base/base/IPv4andIPv6.h | 6 +- programs/benchmark/Benchmark.cpp | 3 +- programs/format/Format.cpp | 3 +- programs/server/Server.cpp | 1 + src/Access/Common/QuotaDefs.cpp | 1 + src/Access/IAccessStorage.h | 2 + src/Access/User.cpp | 1 + .../AggregateFunctionRetention.cpp | 8 +-- .../AggregateFunctionSequenceMatch.cpp | 2 + src/Analyzer/Passes/QueryAnalysisPass.cpp | 1 + src/Backups/BackupEntriesCollector.cpp | 1 + src/Backups/BackupIO_Default.h | 1 + src/Backups/BackupIO_Disk.h | 2 + src/Backups/BackupIO_S3.h | 1 + src/Client/ClientBase.cpp | 9 +-- src/Client/LocalConnection.cpp | 5 +- src/Common/HTTPConnectionPool.cpp | 11 ++-- src/Common/Scheduler/IResourceManager.h | 3 + src/Common/Scheduler/ResouceLink.cpp | 25 ++++++++ src/Common/Scheduler/ResourceGuard.h | 3 +- src/Common/Scheduler/ResourceLink.h | 24 ++----- src/Common/SymbolIndex.cpp | 4 +- src/Common/ThreadStatus.h | 6 +- src/Common/formatIPv6.h | 7 +- ..._proxy_configuration_resolver_provider.cpp | 2 + src/Core/BaseSettings.h | 64 ------------------- src/Core/BaseSettingsProgramOptions.h | 60 +++++++++++++++++ src/Core/LogsLevel.h | 16 +++++ src/Core/SettingsEnums.h | 19 ++---- src/Daemon/SentryWriter.cpp | 1 + src/DataTypes/DataTypeDomainBool.cpp | 5 +- src/DataTypes/DataTypeFactory.h | 1 - src/DataTypes/DataTypeTuple.cpp | 2 +- src/DataTypes/DataTypeTuple.h | 2 +- src/DataTypes/IDataType.cpp | 2 +- src/DataTypes/IDataType.h | 9 ++- src/DataTypes/ObjectUtils.h | 1 + .../Serializations/SerializationInfo.h | 26 ++++---- .../SerializationInfoSettings.h | 14 ++++ .../gtest_DataType_deserializeAsText.cpp | 1 + src/Databases/DatabaseAtomic.cpp | 1 + src/Databases/DatabaseMemory.cpp | 1 + src/Databases/DatabaseOnDisk.cpp | 3 +- src/Databases/DatabaseReplicated.cpp | 1 + src/Databases/DatabasesCommon.cpp | 1 + src/Databases/IDatabase.cpp | 9 +-- .../ObjectStorages/DiskObjectStorage.cpp | 1 + src/Formats/FormatFactory.h | 1 + src/Formats/FormatSchemaInfo.h | 1 + src/Formats/ReadSchemaUtils.cpp | 1 + src/Formats/SchemaInferenceUtils.h | 6 ++ src/Functions/DateTimeTransforms.h | 1 + src/Functions/FunctionJoinGet.cpp | 7 +- .../FunctionsStringHashFixedString.cpp | 2 +- src/Functions/appendTrailingCharIfAbsent.cpp | 3 +- src/Functions/array/arrayIntersect.cpp | 1 + src/Functions/dateDiff.cpp | 2 +- src/Functions/date_trunc.cpp | 3 +- src/Functions/hasColumnInTable.cpp | 1 + .../keyvaluepair/ArgumentExtractor.h | 1 + src/Functions/tupleConcat.cpp | 2 + src/IO/CachedInMemoryReadBufferFromFile.h | 3 +- src/IO/ReadHelpers.h | 9 ++- src/IO/S3/copyS3File.h | 1 + src/IO/examples/read_buffer_from_hdfs.cpp | 2 + src/Interpreters/ActionLocksManager.cpp | 1 + src/Interpreters/ActionsVisitor.cpp | 1 + src/Interpreters/ActionsVisitor.h | 5 +- src/Interpreters/AsynchronousInsertQueue.cpp | 1 + src/Interpreters/CollectJoinOnKeysVisitor.h | 7 +- src/Interpreters/Context.h | 15 ++++- src/Interpreters/DDLTask.cpp | 3 +- src/Interpreters/DatabaseCatalog.cpp | 40 ++++++++++++ src/Interpreters/DatabaseCatalog.h | 64 ------------------- src/Interpreters/IKeyValueEntity.h | 1 + .../InJoinSubqueriesPreprocessor.cpp | 1 + src/Interpreters/InterpreterCheckQuery.cpp | 1 + .../InterpreterCreateIndexQuery.cpp | 5 +- src/Interpreters/InterpreterDescribeQuery.cpp | 1 + .../InterpreterDropIndexQuery.cpp | 1 + src/Interpreters/InterpreterDropQuery.cpp | 1 + src/Interpreters/InterpreterExistsQuery.cpp | 1 + src/Interpreters/InterpreterInsertQuery.cpp | 1 + .../InterpreterKillQueryQuery.cpp | 1 + src/Interpreters/InterpreterOptimizeQuery.cpp | 1 + src/Interpreters/InterpreterRenameQuery.cpp | 1 + .../InterpreterShowCreateQuery.cpp | 1 + .../InterpreterShowFunctionsQuery.cpp | 6 +- src/Interpreters/InterpreterUndropQuery.cpp | 1 + src/Interpreters/InterpreterWatchQuery.cpp | 3 +- src/Interpreters/JoinedTables.cpp | 1 + ...writeSumFunctionWithSumAndCountVisitor.cpp | 2 + src/Interpreters/SystemLog.cpp | 1 + src/Interpreters/SystemLog.h | 3 +- src/Interpreters/TableNameHints.h | 39 +++++++++++ src/Interpreters/executeDDLQueryOnCluster.cpp | 1 + src/Interpreters/getTableExpressions.cpp | 1 + src/Interpreters/interpretSubquery.cpp | 5 +- src/Interpreters/loadMetadata.cpp | 5 +- src/Planner/PlannerJoinTree.cpp | 1 + .../Impl/JSONColumnsBlockInputFormatBase.h | 1 + src/Processors/Formats/Impl/Parquet/Write.h | 1 + .../CreateSetAndFilterOnTheFlyTransform.cpp | 2 + .../getSourceFromASTInsertQuery.cpp | 1 + src/Server/GRPCServer.cpp | 1 + src/Server/MySQLHandler.cpp | 5 +- src/Server/PrometheusMetricsWriter.cpp | 2 + .../Distributed/DistributedSettings.cpp | 7 +- src/Storages/ExecutableSettings.cpp | 6 +- src/Storages/ExecutableSettings.h | 1 + src/Storages/FileLog/StorageFileLog.cpp | 1 + .../extractZkPathFromCreateQuery.cpp | 7 +- src/Storages/RocksDB/StorageSystemRocksDB.cpp | 1 + src/Storages/S3Queue/StorageS3Queue.cpp | 3 +- src/Storages/StorageMaterializedView.cpp | 7 +- src/Storages/StorageNull.cpp | 3 +- src/Storages/System/StorageSystemClusters.cpp | 1 + src/Storages/System/StorageSystemColumns.cpp | 1 + .../System/StorageSystemDatabases.cpp | 1 + .../System/StorageSystemDistributionQueue.cpp | 1 + .../StorageSystemDroppedTablesParts.cpp | 1 + src/Storages/System/StorageSystemGraphite.cpp | 1 + .../System/StorageSystemMutations.cpp | 1 + .../System/StorageSystemPartsBase.cpp | 1 + src/Storages/System/StorageSystemTables.cpp | 1 + .../StorageSystemZooKeeperConnection.cpp | 2 + src/Storages/buildQueryTreeForShard.cpp | 1 + src/Storages/getStructureOfRemoteTable.cpp | 1 + src/TableFunctions/TableFunctionMerge.cpp | 1 + 129 files changed, 439 insertions(+), 266 deletions(-) create mode 100644 src/Common/Scheduler/ResouceLink.cpp create mode 100644 src/Core/BaseSettingsProgramOptions.h create mode 100644 src/Core/LogsLevel.h create mode 100644 src/DataTypes/Serializations/SerializationInfoSettings.h create mode 100644 src/Interpreters/TableNameHints.h diff --git a/base/base/IPv4andIPv6.h b/base/base/IPv4andIPv6.h index e2f93b54124..9b1e518c161 100644 --- a/base/base/IPv4andIPv6.h +++ b/base/base/IPv4andIPv6.h @@ -1,8 +1,7 @@ #pragma once -#include #include -#include +#include #include namespace DB @@ -62,7 +61,8 @@ namespace std { size_t operator()(const DB::IPv6 & x) const { - return std::hash{}(std::string_view(reinterpret_cast(&x.toUnderType()), IPV6_BINARY_LENGTH)); + return std::hash{}( + std::string_view(reinterpret_cast(&x.toUnderType()), sizeof(DB::IPv6::UnderlyingType))); } }; diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 45dadfef774..eecc352d073 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -34,6 +34,7 @@ #include #include #include +#include /** A tool for evaluating ClickHouse performance. @@ -623,7 +624,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ; Settings settings; - settings.addProgramOptions(desc); + addProgramOptions(settings, desc); boost::program_options::variables_map options; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 50f801f2560..fc73eda6815 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -102,7 +103,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) { std::string_view name = field.getName(); if (name == "max_parser_depth" || name == "max_query_size") - cmd_settings.addProgramOption(desc, name, field); + addProgramOption(cmd_settings, desc, name, field); } boost::program_options::variables_map options; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b67a4eccd15..59dadd14b20 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Access/Common/QuotaDefs.cpp b/src/Access/Common/QuotaDefs.cpp index 04c16a562d2..2a22b80f65d 100644 --- a/src/Access/Common/QuotaDefs.cpp +++ b/src/Access/Common/QuotaDefs.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index ad78bf92e02..b00c8bac849 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -13,6 +13,8 @@ #include #include +#include + namespace Poco { class Logger; } namespace Poco::Net { class IPAddress; } diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 91477e5f3aa..39930c9cf76 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -1,4 +1,5 @@ #include +#include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionRetention.cpp b/src/AggregateFunctions/AggregateFunctionRetention.cpp index a004f3527a2..5eaa1a7a39c 100644 --- a/src/AggregateFunctions/AggregateFunctionRetention.cpp +++ b/src/AggregateFunctions/AggregateFunctionRetention.cpp @@ -1,16 +1,16 @@ #include +#include #include - -#include #include #include #include #include #include #include -#include +#include -#include +#include +#include namespace DB diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp index ff9259e3aac..bc3d4da5e39 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp +++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp @@ -10,6 +10,8 @@ #include #include #include +#include + #include #include diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 1da5d86edf3..d8a196ee3d4 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include #include diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 5c0b80aa894..c71ce195388 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Backups/BackupIO_Default.h b/src/Backups/BackupIO_Default.h index 639293f22d9..4dba0324703 100644 --- a/src/Backups/BackupIO_Default.h +++ b/src/Backups/BackupIO_Default.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h index 575ec3f5707..3d3253877bd 100644 --- a/src/Backups/BackupIO_Disk.h +++ b/src/Backups/BackupIO_Disk.h @@ -1,7 +1,9 @@ #pragma once #include +#include #include + #include diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h index 8ab8e1fb566..57108d122ea 100644 --- a/src/Backups/BackupIO_S3.h +++ b/src/Backups/BackupIO_S3.h @@ -4,6 +4,7 @@ #if USE_AWS_S3 #include +#include #include #include #include diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index d561a64895b..582dc2c9e6b 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -2714,9 +2715,9 @@ private: void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments) { if (allow_repeated_settings) - cmd_settings.addProgramOptionsAsMultitokens(options_description.main_description.value()); + addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value()); else - cmd_settings.addProgramOptions(options_description.main_description.value()); + addProgramOptions(cmd_settings, options_description.main_description.value()); if (allow_merge_tree_settings) { @@ -2737,9 +2738,9 @@ void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, return; if (allow_repeated_settings) - cmd_merge_tree_settings.addProgramOptionAsMultitoken(main_options, name, setting); + addProgramOptionAsMultitoken(cmd_merge_tree_settings, main_options, name, setting); else - cmd_merge_tree_settings.addProgramOption(main_options, name, setting); + addProgramOption(cmd_merge_tree_settings, main_options, name, setting); }; const auto & setting_name = setting.getName(); diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index dbb115f44ef..afcaa4d6098 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -1,13 +1,14 @@ #include "LocalConnection.h" +#include +#include #include #include #include -#include #include +#include #include #include #include -#include namespace DB diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index a21438a11a2..f729b8ea8d0 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -10,14 +10,15 @@ #include #include -#include -#include -#include #include +#include +#include +#include +#include +#include #include -#include -#include +#include #include "config.h" diff --git a/src/Common/Scheduler/IResourceManager.h b/src/Common/Scheduler/IResourceManager.h index 0af74e00960..8a7077ac3d5 100644 --- a/src/Common/Scheduler/IResourceManager.h +++ b/src/Common/Scheduler/IResourceManager.h @@ -12,6 +12,9 @@ namespace DB { +class ISchedulerNode; +using SchedulerNodePtr = std::shared_ptr; + /* * Instance of derived class holds everything required for resource consumption, * including resources currently registered at `SchedulerRoot`. This is required to avoid diff --git a/src/Common/Scheduler/ResouceLink.cpp b/src/Common/Scheduler/ResouceLink.cpp new file mode 100644 index 00000000000..2da5dba62dc --- /dev/null +++ b/src/Common/Scheduler/ResouceLink.cpp @@ -0,0 +1,25 @@ +#include +#include +#include + +namespace DB +{ +void ResourceLink::adjust(ResourceCost estimated_cost, ResourceCost real_cost) const +{ + if (queue) + queue->adjustBudget(estimated_cost, real_cost); +} + +void ResourceLink::consumed(ResourceCost cost) const +{ + if (queue) + queue->consumeBudget(cost); +} + +void ResourceLink::accumulate(DB::ResourceCost cost) const +{ + if (queue) + queue->accumulateBudget(cost); +} +} + diff --git a/src/Common/Scheduler/ResourceGuard.h b/src/Common/Scheduler/ResourceGuard.h index 50f665a384b..3c29f588fba 100644 --- a/src/Common/Scheduler/ResourceGuard.h +++ b/src/Common/Scheduler/ResourceGuard.h @@ -2,9 +2,10 @@ #include +#include +#include #include #include -#include #include #include diff --git a/src/Common/Scheduler/ResourceLink.h b/src/Common/Scheduler/ResourceLink.h index 108f51ac399..450d9bc1efa 100644 --- a/src/Common/Scheduler/ResourceLink.h +++ b/src/Common/Scheduler/ResourceLink.h @@ -2,12 +2,10 @@ #include -#include -#include - - namespace DB { +class ISchedulerQueue; +using ResourceCost = Int64; /* * Everything required for resource consumption. Connection to a specific resource queue. @@ -17,23 +15,11 @@ struct ResourceLink ISchedulerQueue * queue = nullptr; bool operator==(const ResourceLink &) const = default; - void adjust(ResourceCost estimated_cost, ResourceCost real_cost) const - { - if (queue) - queue->adjustBudget(estimated_cost, real_cost); - } + void adjust(ResourceCost estimated_cost, ResourceCost real_cost) const; - void consumed(ResourceCost cost) const - { - if (queue) - queue->consumeBudget(cost); - } + void consumed(ResourceCost cost) const; - void accumulate(ResourceCost cost) const - { - if (queue) - queue->accumulateBudget(cost); - } + void accumulate(ResourceCost cost) const; }; } diff --git a/src/Common/SymbolIndex.cpp b/src/Common/SymbolIndex.cpp index 8dde617fc74..a2d58fc7e8c 100644 --- a/src/Common/SymbolIndex.cpp +++ b/src/Common/SymbolIndex.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include @@ -11,8 +13,6 @@ #include -#include - /** ELF object can contain three different places with symbol names and addresses: diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 77386af6c2b..48b52f8aa6e 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -1,12 +1,12 @@ #pragma once -#include -#include +#include #include +#include +#include #include #include #include -#include #include diff --git a/src/Common/formatIPv6.h b/src/Common/formatIPv6.h index fa7a9592c48..3451eda6b3c 100644 --- a/src/Common/formatIPv6.h +++ b/src/Common/formatIPv6.h @@ -1,13 +1,12 @@ #pragma once -#include -#include #include +#include #include #include -#include -#include #include +#include +#include #include constexpr size_t IPV4_BINARY_LENGTH = 4; diff --git a/src/Common/tests/gtest_proxy_configuration_resolver_provider.cpp b/src/Common/tests/gtest_proxy_configuration_resolver_provider.cpp index 9a7447b02e4..d5d6f86f661 100644 --- a/src/Common/tests/gtest_proxy_configuration_resolver_provider.cpp +++ b/src/Common/tests/gtest_proxy_configuration_resolver_provider.cpp @@ -4,6 +4,8 @@ #include #include +#include + using ConfigurationPtr = Poco::AutoPtr; class ProxyConfigurationResolverProviderTests : public ::testing::Test diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index 6f3245c83e8..7191038a4ce 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -7,7 +7,6 @@ #include #include #include -#include namespace boost::program_options @@ -129,18 +128,6 @@ public: std::conditional_t custom_setting; }; - /// Adds program options to set the settings from a command line. - /// (Don't forget to call notify() on the `variables_map` after parsing it!) - void addProgramOptions(boost::program_options::options_description & options); - - /// Adds program options as to set the settings from a command line. - /// Allows to set one setting multiple times, the last value will be used. - /// (Don't forget to call notify() on the `variables_map` after parsing it!) - void addProgramOptionsAsMultitokens(boost::program_options::options_description & options); - - void addProgramOption(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field); - void addProgramOptionAsMultitoken(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field); - enum SkipFlags { SKIP_NONE = 0, @@ -561,57 +548,6 @@ String BaseSettings::toString() const return out.str(); } -template -void BaseSettings::addProgramOptions(boost::program_options::options_description & options) -{ - const auto & settings_to_aliases = TTraits::settingsToAliases(); - for (const auto & field : all()) - { - std::string_view name = field.getName(); - addProgramOption(options, name, field); - - if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end()) - { - for (const auto alias : it->second) - addProgramOption(options, alias, field); - } - } -} - -template -void BaseSettings::addProgramOptionsAsMultitokens(boost::program_options::options_description & options) -{ - const auto & settings_to_aliases = TTraits::settingsToAliases(); - for (const auto & field : all()) - { - std::string_view name = field.getName(); - addProgramOptionAsMultitoken(options, name, field); - - if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end()) - { - for (const auto alias : it->second) - addProgramOptionAsMultitoken(options, alias, field); - } - } -} - - -template -void BaseSettings::addProgramOption(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field) -{ - auto on_program_option = boost::function1([this, name](const std::string & value) { set(name, value); }); - options.add(boost::shared_ptr(new boost::program_options::option_description( - name.data(), boost::program_options::value()->composing()->notifier(on_program_option), field.getDescription()))); -} - -template -void BaseSettings::addProgramOptionAsMultitoken(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field) -{ - auto on_program_option = boost::function1([this, name](const Strings & values) { set(name, values.back()); }); - options.add(boost::shared_ptr(new boost::program_options::option_description( - name.data(), boost::program_options::value()->multitoken()->composing()->notifier(on_program_option), field.getDescription()))); -} - template bool operator==(const BaseSettings & left, const BaseSettings & right) { diff --git a/src/Core/BaseSettingsProgramOptions.h b/src/Core/BaseSettingsProgramOptions.h new file mode 100644 index 00000000000..6c8166fc119 --- /dev/null +++ b/src/Core/BaseSettingsProgramOptions.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +#include + +namespace DB +{ + +template +void addProgramOptionAsMultitoken(T &cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field) +{ + auto on_program_option = boost::function1([&cmd_settings, name](const Strings & values) { cmd_settings.set(name, values.back()); }); + options.add(boost::shared_ptr(new boost::program_options::option_description( + name.data(), boost::program_options::value()->multitoken()->composing()->notifier(on_program_option), field.getDescription()))); +} + +template +void addProgramOptionsAsMultitokens(T &cmd_settings, boost::program_options::options_description & options) +{ + const auto & settings_to_aliases = T::Traits::settingsToAliases(); + for (const auto & field : cmd_settings.all()) + { + std::string_view name = field.getName(); + addProgramOptionAsMultitoken(cmd_settings, options, name, field); + + if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end()) + for (const auto alias : it->second) + addProgramOptionAsMultitoken(cmd_settings, options, alias, field); + } +} + +/// Adds program options to set the settings from a command line. +/// (Don't forget to call notify() on the `variables_map` after parsing it!) +template +void addProgramOption(T &cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field) +{ + auto on_program_option = boost::function1([&cmd_settings, name](const std::string & value) { cmd_settings.set(name, value); }); + options.add(boost::shared_ptr(new boost::program_options::option_description( + name.data(), boost::program_options::value()->composing()->notifier(on_program_option), field.getDescription()))); +} + +template +void addProgramOptions(T &cmd_settings, boost::program_options::options_description & options) +{ + const auto & settings_to_aliases = T::Traits::settingsToAliases(); + for (const auto & field : cmd_settings.all()) + { + std::string_view name = field.getName(); + addProgramOption(cmd_settings, options, name, field); + + if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end()) + for (const auto alias : it->second) + addProgramOption(cmd_settings, options, alias, field); + } +} + + +} diff --git a/src/Core/LogsLevel.h b/src/Core/LogsLevel.h new file mode 100644 index 00000000000..95721820d9c --- /dev/null +++ b/src/Core/LogsLevel.h @@ -0,0 +1,16 @@ +#pragma once + +namespace DB +{ +enum class LogsLevel +{ + none = 0, /// Disable + fatal, + error, + warning, + information, + debug, + trace, + test, +}; +} diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 22fcf0389d8..421712ed868 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -1,12 +1,13 @@ #pragma once -#include #include -#include +#include +#include #include #include -#include #include +#include +#include namespace DB @@ -90,18 +91,6 @@ DECLARE_SETTING_ENUM_WITH_RENAME(IntervalOutputFormat, FormatSettings::IntervalO DECLARE_SETTING_ENUM_WITH_RENAME(ParquetVersion, FormatSettings::ParquetVersion) -enum class LogsLevel -{ - none = 0, /// Disable - fatal, - error, - warning, - information, - debug, - trace, - test, -}; - DECLARE_SETTING_ENUM(LogsLevel) diff --git a/src/Daemon/SentryWriter.cpp b/src/Daemon/SentryWriter.cpp index 192e9952b9a..0fa8b99a6f7 100644 --- a/src/Daemon/SentryWriter.cpp +++ b/src/Daemon/SentryWriter.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "config.h" #include diff --git a/src/DataTypes/DataTypeDomainBool.cpp b/src/DataTypes/DataTypeDomainBool.cpp index 245c5495299..3d19b6262d8 100644 --- a/src/DataTypes/DataTypeDomainBool.cpp +++ b/src/DataTypes/DataTypeDomainBool.cpp @@ -1,6 +1,7 @@ -#include -#include #include +#include +#include +#include namespace DB { diff --git a/src/DataTypes/DataTypeFactory.h b/src/DataTypes/DataTypeFactory.h index a2aeb6f3646..4727cb3ae5c 100644 --- a/src/DataTypes/DataTypeFactory.h +++ b/src/DataTypes/DataTypeFactory.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index eb218d8efb7..5bbd79160d4 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -346,7 +346,7 @@ SerializationPtr DataTypeTuple::getSerialization(const SerializationInfo & info) return std::make_shared(std::move(serializations), have_explicit_names); } -MutableSerializationInfoPtr DataTypeTuple::createSerializationInfo(const SerializationInfo::Settings & settings) const +MutableSerializationInfoPtr DataTypeTuple::createSerializationInfo(const SerializationInfoSettings & settings) const { MutableSerializationInfos infos; infos.reserve(elems.size()); diff --git a/src/DataTypes/DataTypeTuple.h b/src/DataTypes/DataTypeTuple.h index 4e5a0c1b33c..15561fe4286 100644 --- a/src/DataTypes/DataTypeTuple.h +++ b/src/DataTypes/DataTypeTuple.h @@ -58,7 +58,7 @@ public: SerializationPtr doGetDefaultSerialization() const override; SerializationPtr getSerialization(const SerializationInfo & info) const override; - MutableSerializationInfoPtr createSerializationInfo(const SerializationInfo::Settings & settings) const override; + MutableSerializationInfoPtr createSerializationInfo(const SerializationInfoSettings & settings) const override; SerializationInfoPtr getSerializationInfo(const IColumn & column) const override; const DataTypePtr & getElement(size_t i) const { return elems[i]; } diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index 40915418aea..344b81be960 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -202,7 +202,7 @@ void IDataType::setCustomization(DataTypeCustomDescPtr custom_desc_) const custom_serialization = std::move(custom_desc_->serialization); } -MutableSerializationInfoPtr IDataType::createSerializationInfo(const SerializationInfo::Settings & settings) const +MutableSerializationInfoPtr IDataType::createSerializationInfo(const SerializationInfoSettings & settings) const { return std::make_shared(ISerialization::Kind::DEFAULT, settings); } diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 4403e3d9bd4..eaf798a3017 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -7,8 +7,6 @@ #include #include #include -#include - namespace DB { @@ -38,6 +36,11 @@ struct DataTypeWithConstInfo using DataTypesWithConstInfo = std::vector; +class SerializationInfo; +using SerializationInfoPtr = std::shared_ptr; +using MutableSerializationInfoPtr = std::shared_ptr; +struct SerializationInfoSettings; + /** Properties of data type. * * Contains methods for getting serialization instances. @@ -117,7 +120,7 @@ public: Names getSubcolumnNames() const; - virtual MutableSerializationInfoPtr createSerializationInfo(const SerializationInfo::Settings & settings) const; + virtual MutableSerializationInfoPtr createSerializationInfo(const SerializationInfoSettings & settings) const; virtual SerializationInfoPtr getSerializationInfo(const IColumn & column) const; /// TODO: support more types. diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 2bfcaae09ca..9897f7d20f9 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -13,6 +13,7 @@ namespace DB struct StorageSnapshot; using StorageSnapshotPtr = std::shared_ptr; +class ColumnsDescription; /// Returns number of dimensions in Array type. 0 if type is not array. size_t getNumberOfDimensions(const IDataType & type); diff --git a/src/DataTypes/Serializations/SerializationInfo.h b/src/DataTypes/Serializations/SerializationInfo.h index 3d8f4f1d00c..5a900a5521c 100644 --- a/src/DataTypes/Serializations/SerializationInfo.h +++ b/src/DataTypes/Serializations/SerializationInfo.h @@ -2,6 +2,8 @@ #include #include +#include + #include @@ -28,6 +30,8 @@ constexpr auto SERIALIZATION_INFO_VERSION = 0; class SerializationInfo { public: + using Settings = SerializationInfoSettings; + struct Data { size_t num_rows = 0; @@ -38,16 +42,8 @@ public: void addDefaults(size_t length); }; - struct Settings - { - const double ratio_of_defaults_for_sparse = 1.0; - const bool choose_kind = false; - - bool isAlwaysDefault() const { return ratio_of_defaults_for_sparse >= 1.0; } - }; - - SerializationInfo(ISerialization::Kind kind_, const Settings & settings_); - SerializationInfo(ISerialization::Kind kind_, const Settings & settings_, const Data & data_); + SerializationInfo(ISerialization::Kind kind_, const SerializationInfoSettings & settings_); + SerializationInfo(ISerialization::Kind kind_, const SerializationInfoSettings & settings_, const Data & data_); virtual ~SerializationInfo() = default; @@ -64,7 +60,7 @@ public: virtual std::shared_ptr createWithType( const IDataType & old_type, const IDataType & new_type, - const Settings & new_settings) const; + const SerializationInfoSettings & new_settings) const; virtual void serialializeKindBinary(WriteBuffer & out) const; virtual void deserializeFromKindsBinary(ReadBuffer & in); @@ -73,14 +69,14 @@ public: virtual void fromJSON(const Poco::JSON::Object & object); void setKind(ISerialization::Kind kind_) { kind = kind_; } - const Settings & getSettings() const { return settings; } + const SerializationInfoSettings & getSettings() const { return settings; } const Data & getData() const { return data; } ISerialization::Kind getKind() const { return kind; } - static ISerialization::Kind chooseKind(const Data & data, const Settings & settings); + static ISerialization::Kind chooseKind(const Data & data, const SerializationInfoSettings & settings); protected: - const Settings settings; + const SerializationInfoSettings settings; ISerialization::Kind kind; Data data; @@ -96,7 +92,7 @@ using MutableSerializationInfos = std::vector; class SerializationInfoByName : public std::map { public: - using Settings = SerializationInfo::Settings; + using Settings = SerializationInfoSettings; SerializationInfoByName() = default; SerializationInfoByName(const NamesAndTypesList & columns, const Settings & settings); diff --git a/src/DataTypes/Serializations/SerializationInfoSettings.h b/src/DataTypes/Serializations/SerializationInfoSettings.h new file mode 100644 index 00000000000..26f2c344bea --- /dev/null +++ b/src/DataTypes/Serializations/SerializationInfoSettings.h @@ -0,0 +1,14 @@ +#pragma once + +namespace DB +{ + +struct SerializationInfoSettings +{ + const double ratio_of_defaults_for_sparse = 1.0; + const bool choose_kind = false; + + bool isAlwaysDefault() const { return ratio_of_defaults_for_sparse >= 1.0; } +}; + +} diff --git a/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp b/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp index 0373e55a62d..bf5337c89da 100644 --- a/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp +++ b/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 0bf7c8af4b4..6f3f1151153 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 794eebbc399..4ff7b3c7f2b 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index dcfc1916450..896baa561b4 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -5,8 +5,9 @@ #include #include #include -#include #include +#include +#include #include #include #include diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 3b6a712510d..59b3e52e139 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index f8d6ad69ba8..d40e5f98aaa 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/IDatabase.cpp b/src/Databases/IDatabase.cpp index ae8fc58bf89..95d671d1960 100644 --- a/src/Databases/IDatabase.cpp +++ b/src/Databases/IDatabase.cpp @@ -1,11 +1,12 @@ #include #include -#include -#include -#include #include -#include +#include +#include +#include #include +#include +#include namespace CurrentMetrics diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 696a0ef7416..9d20427e0d5 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 46c1b8ddcdd..93f67e83831 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -23,6 +23,7 @@ namespace DB class Block; struct Settings; struct FormatFactorySettings; +struct ReadSettings; class ReadBuffer; class WriteBuffer; diff --git a/src/Formats/FormatSchemaInfo.h b/src/Formats/FormatSchemaInfo.h index e8758c3f761..0dd28699896 100644 --- a/src/Formats/FormatSchemaInfo.h +++ b/src/Formats/FormatSchemaInfo.h @@ -8,6 +8,7 @@ namespace DB { class Context; +class Block; /// Extracts information about where the format schema file is from passed context and keep it. class FormatSchemaInfo diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp index b05b768899b..735b536986d 100644 --- a/src/Formats/ReadSchemaUtils.cpp +++ b/src/Formats/ReadSchemaUtils.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Formats/SchemaInferenceUtils.h b/src/Formats/SchemaInferenceUtils.h index b492d9b22b6..bcf3d194825 100644 --- a/src/Formats/SchemaInferenceUtils.h +++ b/src/Formats/SchemaInferenceUtils.h @@ -3,9 +3,15 @@ #include #include +#include + namespace DB { +class Block; +class NamesAndTypesList; +using NamesAndTypesLists = std::vector; + /// Struct with some additional information about inferred types for JSON formats. struct JSONInferenceInfo { diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index f5072de24f3..558c309007c 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index 5602c88c60e..085c4db3f57 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -1,12 +1,13 @@ #include +#include #include #include -#include -#include #include +#include +#include +#include #include #include -#include namespace DB diff --git a/src/Functions/FunctionsStringHashFixedString.cpp b/src/Functions/FunctionsStringHashFixedString.cpp index bbb02de6c9f..f4160badd37 100644 --- a/src/Functions/FunctionsStringHashFixedString.cpp +++ b/src/Functions/FunctionsStringHashFixedString.cpp @@ -274,7 +274,7 @@ public: const typename ColumnIPv6::Container & data = col_from_ip->getData(); const auto size = col_from_ip->size(); auto & chars_to = col_to->getChars(); - const auto length = IPV6_BINARY_LENGTH; + const auto length = sizeof(IPv6::UnderlyingType); chars_to.resize(size * Impl::length); for (size_t i = 0; i < size; ++i) { diff --git a/src/Functions/appendTrailingCharIfAbsent.cpp b/src/Functions/appendTrailingCharIfAbsent.cpp index 7ff35e599be..a5554171aaa 100644 --- a/src/Functions/appendTrailingCharIfAbsent.cpp +++ b/src/Functions/appendTrailingCharIfAbsent.cpp @@ -1,9 +1,10 @@ #include -#include #include #include #include #include +#include +#include namespace DB diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index ffc7480219e..209441eb301 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index f75e6eb4fc8..8f318d1bc55 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -6,7 +6,7 @@ #include #include #include - +#include #include #include #include diff --git a/src/Functions/date_trunc.cpp b/src/Functions/date_trunc.cpp index f64848e9185..8493df17a2f 100644 --- a/src/Functions/date_trunc.cpp +++ b/src/Functions/date_trunc.cpp @@ -1,9 +1,10 @@ #include -#include #include +#include #include #include #include +#include #include #include diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp index 66ed515e490..48783a672e2 100644 --- a/src/Functions/hasColumnInTable.cpp +++ b/src/Functions/hasColumnInTable.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include diff --git a/src/Functions/keyvaluepair/ArgumentExtractor.h b/src/Functions/keyvaluepair/ArgumentExtractor.h index e6538584d01..6ff8aa36f13 100644 --- a/src/Functions/keyvaluepair/ArgumentExtractor.h +++ b/src/Functions/keyvaluepair/ArgumentExtractor.h @@ -4,6 +4,7 @@ #include #include +#include #include namespace DB diff --git a/src/Functions/tupleConcat.cpp b/src/Functions/tupleConcat.cpp index 0556f4181e6..c48e4d61463 100644 --- a/src/Functions/tupleConcat.cpp +++ b/src/Functions/tupleConcat.cpp @@ -4,6 +4,8 @@ #include #include +#include + namespace DB { namespace ErrorCodes diff --git a/src/IO/CachedInMemoryReadBufferFromFile.h b/src/IO/CachedInMemoryReadBufferFromFile.h index 300c2e82386..a0d07486359 100644 --- a/src/IO/CachedInMemoryReadBufferFromFile.h +++ b/src/IO/CachedInMemoryReadBufferFromFile.h @@ -1,7 +1,8 @@ #pragma once -#include #include +#include +#include namespace DB { diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 49530f4787a..ca568c469b4 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -154,9 +154,12 @@ inline void readIPv6Binary(IPv6 & ip, ReadBuffer & buf) size_t size = 0; readVarUInt(size, buf); - if (size != IPV6_BINARY_LENGTH) - throw Exception(ErrorCodes::SIZE_OF_FIXED_STRING_DOESNT_MATCH, - "Size of the string {} doesn't match size of binary IPv6 {}", size, IPV6_BINARY_LENGTH); + if (size != sizeof(IPv6::UnderlyingType)) + throw Exception( + ErrorCodes::SIZE_OF_FIXED_STRING_DOESNT_MATCH, + "Size of the string {} doesn't match size of binary IPv6 {}", + size, + sizeof(IPv6::UnderlyingType)); buf.readStrict(reinterpret_cast(&ip.toUnderType()), size); } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 093d26ba7bb..5eb6f702473 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -14,6 +14,7 @@ namespace DB { +struct ReadSettings; class SeekableReadBuffer; using CreateReadBuffer = std::function()>; diff --git a/src/IO/examples/read_buffer_from_hdfs.cpp b/src/IO/examples/read_buffer_from_hdfs.cpp index 977dd2ae227..c499542fedb 100644 --- a/src/IO/examples/read_buffer_from_hdfs.cpp +++ b/src/IO/examples/read_buffer_from_hdfs.cpp @@ -6,6 +6,8 @@ #include #include +#include + using namespace DB; int main() diff --git a/src/Interpreters/ActionLocksManager.cpp b/src/Interpreters/ActionLocksManager.cpp index 65f13ebd66c..28803a94c80 100644 --- a/src/Interpreters/ActionLocksManager.cpp +++ b/src/Interpreters/ActionLocksManager.cpp @@ -1,5 +1,6 @@ #include "ActionLocksManager.h" #include +#include #include #include diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index f9a58c8c3ca..16e2449206d 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index 1e2ebaf6d87..643bf50ecd4 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -1,13 +1,14 @@ #pragma once #include +#include +#include #include #include #include #include #include -#include -#include +#include namespace DB { diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 9234d052d97..1c6039bbbf7 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/CollectJoinOnKeysVisitor.h b/src/Interpreters/CollectJoinOnKeysVisitor.h index 194ec01bcd5..ff10db3beb5 100644 --- a/src/Interpreters/CollectJoinOnKeysVisitor.h +++ b/src/Interpreters/CollectJoinOnKeysVisitor.h @@ -1,11 +1,12 @@ #pragma once +#include #include +#include +#include +#include #include #include -#include -#include -#include namespace DB diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index dbd9bbd1ff8..c2d6febc733 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -19,9 +19,8 @@ #include #include #include -#include +#include #include -#include #include #include #include @@ -149,6 +148,18 @@ template class MergeTreeBackgroundExecutor; class AsyncLoader; +struct TemporaryTableHolder; +using TemporaryTablesMapping = std::map>; + +class LoadTask; +using LoadTaskPtr = std::shared_ptr; +using LoadTaskPtrs = std::vector; + +class IClassifier; +using ClassifierPtr = std::shared_ptr; +class IResourceManager; +using ResourceManagerPtr = std::shared_ptr; + /// Scheduling policy can be changed using `background_merges_mutations_scheduling_policy` config option. /// By default concurrent merges are scheduled using "round_robin" to ensure fair and starvation-free operation. /// Previously in heavily overloaded shards big merges could possibly be starved by smaller diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index e10f3ecfbc9..a37b4db029a 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include #include @@ -14,7 +16,6 @@ #include #include #include -#include namespace DB diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index c2f2003aabd..ec6c8b5924f 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -1705,4 +1706,43 @@ DDLGuard::~DDLGuard() releaseTableLock(); } +std::pair TableNameHints::getHintForTable(const String & table_name) const +{ + auto results = this->getHints(table_name, getAllRegisteredNames()); + if (results.empty()) + return getExtendedHintForTable(table_name); + return std::make_pair(database->getDatabaseName(), results[0]); +} + +std::pair TableNameHints::getExtendedHintForTable(const String & table_name) const +{ + /// load all available databases from the DatabaseCatalog instance + auto & database_catalog = DatabaseCatalog::instance(); + auto all_databases = database_catalog.getDatabases(); + + for (const auto & [db_name, db] : all_databases) + { + /// this case should be covered already by getHintForTable + if (db_name == database->getDatabaseName()) + continue; + + TableNameHints hints(db, context); + auto results = hints.getHints(table_name); + + /// if the results are not empty, return the first instance of the table_name + /// and the corresponding database_name that was found. + if (!results.empty()) + return std::make_pair(db_name, results[0]); + } + return {}; +} + +Names TableNameHints::getAllRegisteredNames() const +{ + Names result; + if (database) + for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next()) + result.emplace_back(table_it->name()); + return result; +} } diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 6995fc51941..30b49cf1fed 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -1,13 +1,11 @@ #pragma once #include -#include #include #include #include #include #include -#include #include #include @@ -365,68 +363,6 @@ private: static constexpr time_t DBMS_DEFAULT_DISK_RELOAD_PERIOD_SEC = 5; }; -class TableNameHints : public IHints<> -{ -public: - TableNameHints(ConstDatabasePtr database_, ContextPtr context_) - : context(context_), - database(database_) - { - } - - /// getHintForTable tries to get a hint for the provided table_name in the provided - /// database. If the results are empty, it goes for extended hints for the table - /// with getExtendedHintForTable which looks for the table name in every database that's - /// available in the database catalog. It finally returns a single hint which is the database - /// name and table_name pair which is similar to the table_name provided. Perhaps something to - /// consider is should we return more than one pair of hint? - std::pair getHintForTable(const String & table_name) const - { - auto results = this->getHints(table_name, getAllRegisteredNames()); - if (results.empty()) - return getExtendedHintForTable(table_name); - return std::make_pair(database->getDatabaseName(), results[0]); - } - - /// getExtendedHintsForTable tries to get hint for the given table_name across all - /// the databases that are available in the database catalog. - std::pair getExtendedHintForTable(const String & table_name) const - { - /// load all available databases from the DatabaseCatalog instance - auto & database_catalog = DatabaseCatalog::instance(); - auto all_databases = database_catalog.getDatabases(); - - for (const auto & [db_name, db] : all_databases) - { - /// this case should be covered already by getHintForTable - if (db_name == database->getDatabaseName()) - continue; - - TableNameHints hints(db, context); - auto results = hints.getHints(table_name); - - /// if the results are not empty, return the first instance of the table_name - /// and the corresponding database_name that was found. - if (!results.empty()) - return std::make_pair(db_name, results[0]); - } - return {}; - } - - Names getAllRegisteredNames() const override - { - Names result; - if (database) - for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next()) - result.emplace_back(table_it->name()); - return result; - } - -private: - ContextPtr context; - ConstDatabasePtr database; -}; - /// This class is useful when creating a table or database. /// Usually we create IStorage/IDatabase object first and then add it to IDatabase/DatabaseCatalog. diff --git a/src/Interpreters/IKeyValueEntity.h b/src/Interpreters/IKeyValueEntity.h index d1ceda57f0e..856ce28bae7 100644 --- a/src/Interpreters/IKeyValueEntity.h +++ b/src/Interpreters/IKeyValueEntity.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include diff --git a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index ec4241a2740..3b3ef928b42 100644 --- a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 98a281bd5ad..ae8cef3f102 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index cd2f996c74c..aed4b0587b4 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -1,10 +1,11 @@ -#include -#include #include #include #include +#include #include +#include +#include #include #include #include diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp index 04d44e34fff..87b0eb89302 100644 --- a/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/src/Interpreters/InterpreterDescribeQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp index 025677eeb91..f052aa201f1 100644 --- a/src/Interpreters/InterpreterDropIndexQuery.cpp +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 72aa4cc63e3..a7040709034 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterExistsQuery.cpp b/src/Interpreters/InterpreterExistsQuery.cpp index e4176a44533..14ca9eaed0a 100644 --- a/src/Interpreters/InterpreterExistsQuery.cpp +++ b/src/Interpreters/InterpreterExistsQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 3e8bb268fe7..fc58f7b5098 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 26dae6a1df3..6d6b1085ffb 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp index 1e2eaa50ab1..907a01b0432 100644 --- a/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index 06b6ebc9cbb..b63672f1757 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 9edac1fd8e1..0fca7b64d5a 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterShowFunctionsQuery.cpp b/src/Interpreters/InterpreterShowFunctionsQuery.cpp index 829670d7929..96a730f6527 100644 --- a/src/Interpreters/InterpreterShowFunctionsQuery.cpp +++ b/src/Interpreters/InterpreterShowFunctionsQuery.cpp @@ -1,8 +1,8 @@ -#include -#include - +#include #include #include +#include +#include #include #include diff --git a/src/Interpreters/InterpreterUndropQuery.cpp b/src/Interpreters/InterpreterUndropQuery.cpp index f628a656947..920df3d6aed 100644 --- a/src/Interpreters/InterpreterUndropQuery.cpp +++ b/src/Interpreters/InterpreterUndropQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterWatchQuery.cpp b/src/Interpreters/InterpreterWatchQuery.cpp index 2b68c5d7a10..4937d8660e0 100644 --- a/src/Interpreters/InterpreterWatchQuery.cpp +++ b/src/Interpreters/InterpreterWatchQuery.cpp @@ -12,9 +12,10 @@ limitations under the License. */ #include #include #include +#include +#include #include #include -#include #include #include #include diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 49693332280..5b549a19083 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include diff --git a/src/Interpreters/RewriteSumFunctionWithSumAndCountVisitor.cpp b/src/Interpreters/RewriteSumFunctionWithSumAndCountVisitor.cpp index 2f5e597bdab..8e60603cca9 100644 --- a/src/Interpreters/RewriteSumFunctionWithSumAndCountVisitor.cpp +++ b/src/Interpreters/RewriteSumFunctionWithSumAndCountVisitor.cpp @@ -4,6 +4,8 @@ #include #include +#include + namespace DB { diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index efb2559ce42..db73fe038c0 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index c296b91e24a..e5b79585701 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -1,8 +1,9 @@ #pragma once +#include #include -#include +#include namespace DB { diff --git a/src/Interpreters/TableNameHints.h b/src/Interpreters/TableNameHints.h new file mode 100644 index 00000000000..d3d8d10c50c --- /dev/null +++ b/src/Interpreters/TableNameHints.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +#include + +namespace DB +{ + +class IDatabase; +using ConstDatabasePtr = std::shared_ptr; + +class TableNameHints : public IHints<> +{ +public: + TableNameHints(ConstDatabasePtr database_, ContextPtr context_) : context(context_), database(database_) { } + + /// getHintForTable tries to get a hint for the provided table_name in the provided + /// database. If the results are empty, it goes for extended hints for the table + /// with getExtendedHintForTable which looks for the table name in every database that's + /// available in the database catalog. It finally returns a single hint which is the database + /// name and table_name pair which is similar to the table_name provided. Perhaps something to + /// consider is should we return more than one pair of hint? + std::pair getHintForTable(const String & table_name) const; + + /// getExtendedHintsForTable tries to get hint for the given table_name across all + /// the databases that are available in the database catalog. + std::pair getExtendedHintForTable(const String & table_name) const; + + Names getAllRegisteredNames() const override; + +private: + ContextPtr context; + ConstDatabasePtr database; +}; + +} diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index df8236c11f4..fbcb57b6125 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index 2853be4c05e..cd92d9ecbf3 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/interpretSubquery.cpp b/src/Interpreters/interpretSubquery.cpp index 5f00be07fa5..6b42345f1d6 100644 --- a/src/Interpreters/interpretSubquery.cpp +++ b/src/Interpreters/interpretSubquery.cpp @@ -10,9 +10,10 @@ #include #include -#include -#include #include +#include +#include +#include namespace DB { diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 226472175b3..9c3922b8bda 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -6,11 +6,12 @@ #include #include +#include +#include #include #include -#include -#include #include +#include #include #include diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 8ca8f0f258b..a8d455cf9ee 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -51,6 +51,7 @@ #include #include +#include #include #include #include diff --git a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h index 6d2532b350a..b1fd86bb806 100644 --- a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h +++ b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include diff --git a/src/Processors/Formats/Impl/Parquet/Write.h b/src/Processors/Formats/Impl/Parquet/Write.h index 24733ac276b..f162984fd5e 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.h +++ b/src/Processors/Formats/Impl/Parquet/Write.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Processors/Transforms/CreateSetAndFilterOnTheFlyTransform.cpp b/src/Processors/Transforms/CreateSetAndFilterOnTheFlyTransform.cpp index 83a75318d61..c38e9e97ed5 100644 --- a/src/Processors/Transforms/CreateSetAndFilterOnTheFlyTransform.cpp +++ b/src/Processors/Transforms/CreateSetAndFilterOnTheFlyTransform.cpp @@ -12,6 +12,8 @@ #include #include +#include + namespace DB { diff --git a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp index 8a13973b970..a2a42f27c3f 100644 --- a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index dfae812c620..72135157117 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 9efcebfc72d..ad5cae01540 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -22,10 +23,10 @@ #include #include #include -#include -#include #include +#include #include +#include #if USE_SSL # include diff --git a/src/Server/PrometheusMetricsWriter.cpp b/src/Server/PrometheusMetricsWriter.cpp index d0fdcd61493..85eafbe4808 100644 --- a/src/Server/PrometheusMetricsWriter.cpp +++ b/src/Server/PrometheusMetricsWriter.cpp @@ -1,7 +1,9 @@ #include "PrometheusMetricsWriter.h" #include +#include #include + #include #include "config.h" diff --git a/src/Storages/Distributed/DistributedSettings.cpp b/src/Storages/Distributed/DistributedSettings.cpp index e07b8da34af..1f6aa6c72fa 100644 --- a/src/Storages/Distributed/DistributedSettings.cpp +++ b/src/Storages/Distributed/DistributedSettings.cpp @@ -1,9 +1,12 @@ -#include +#include #include -#include #include +#include +#include #include +#include + namespace DB { diff --git a/src/Storages/ExecutableSettings.cpp b/src/Storages/ExecutableSettings.cpp index dc462350a06..d00e4098181 100644 --- a/src/Storages/ExecutableSettings.cpp +++ b/src/Storages/ExecutableSettings.cpp @@ -1,10 +1,10 @@ #include "ExecutableSettings.h" -#include - +#include #include -#include #include +#include +#include namespace DB { diff --git a/src/Storages/ExecutableSettings.h b/src/Storages/ExecutableSettings.h index 10dbae8ac9f..95627f08d16 100644 --- a/src/Storages/ExecutableSettings.h +++ b/src/Storages/ExecutableSettings.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB { diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index a5f2331a068..7b0cfdf6a6c 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp b/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp index 45d667047af..8ea732b0243 100644 --- a/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp +++ b/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp @@ -1,11 +1,12 @@ -#include -#include #include #include +#include +#include #include #include #include -#include +#include +#include namespace DB diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.cpp b/src/Storages/RocksDB/StorageSystemRocksDB.cpp index eec2f53381f..4406a7c3fd4 100644 --- a/src/Storages/RocksDB/StorageSystemRocksDB.cpp +++ b/src/Storages/RocksDB/StorageSystemRocksDB.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/S3Queue/StorageS3Queue.cpp b/src/Storages/S3Queue/StorageS3Queue.cpp index 6e7ac2b47b8..e4cd166437e 100644 --- a/src/Storages/S3Queue/StorageS3Queue.cpp +++ b/src/Storages/S3Queue/StorageS3Queue.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -24,8 +25,8 @@ #include #include #include -#include +#include namespace fs = std::filesystem; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 9958d65819b..16428d2dc1c 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -5,14 +5,15 @@ #include #include +#include #include +#include #include #include -#include #include -#include +#include #include -#include +#include #include #include diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index 5e4fde99306..7a8852ca2d6 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -2,8 +2,9 @@ #include #include -#include #include +#include +#include #include diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp index 3c01b4717cc..cb8d5caa50c 100644 --- a/src/Storages/System/StorageSystemClusters.cpp +++ b/src/Storages/System/StorageSystemClusters.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 5c96c6502af..8c6d29a3b70 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemDatabases.cpp b/src/Storages/System/StorageSystemDatabases.cpp index f5537b508ba..2351c3c6a2a 100644 --- a/src/Storages/System/StorageSystemDatabases.cpp +++ b/src/Storages/System/StorageSystemDatabases.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index 50c6436f316..e2058448904 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace DB diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.cpp b/src/Storages/System/StorageSystemDroppedTablesParts.cpp index 20baeee1d3b..344c653f7e4 100644 --- a/src/Storages/System/StorageSystemDroppedTablesParts.cpp +++ b/src/Storages/System/StorageSystemDroppedTablesParts.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index eaa386763c2..d8b760e1302 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 60b80e0b0ad..94656008029 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 6bdfdd357e8..812a4e6efbe 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 639c1455b83..edfc7213dcd 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemZooKeeperConnection.cpp b/src/Storages/System/StorageSystemZooKeeperConnection.cpp index 8041370ee92..950e20512c0 100644 --- a/src/Storages/System/StorageSystemZooKeeperConnection.cpp +++ b/src/Storages/System/StorageSystemZooKeeperConnection.cpp @@ -8,6 +8,8 @@ #include #include +#include + namespace DB { diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index c87a1b216ca..41c8f475f03 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp index c545367b63d..26e953c0578 100644 --- a/src/Storages/getStructureOfRemoteTable.cpp +++ b/src/Storages/getStructureOfRemoteTable.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index ff1459ca34b..13b5c167e0e 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include From 824092b6192b043e28cd775284b27ca720d3e7fe Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 19 Mar 2024 18:38:02 +0100 Subject: [PATCH 0243/1165] Updated to use single part upload for single block of small size and fixed test fail --- .../IO/WriteBufferFromAzureBlobStorage.cpp | 20 ++++++++++++++++++- .../AzureBlobStorage/AzureBlobStorageAuth.cpp | 2 +- .../test.py | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index 50763f7f189..f966e431506 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -89,11 +89,25 @@ void WriteBufferFromAzureBlobStorage::execWithRetry(std::function func, void WriteBufferFromAzureBlobStorage::finalizeImpl() { + auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); + + if (buffer_allocation_policy->getBufferNumber() == 1) + { + size_t data_size = size_t(position() - memory.data()); + if (data_size <= max_single_part_upload_size) + { + Azure::Core::IO::MemoryBodyStream memory_stream(reinterpret_cast(memory.data()), data_size); + execWithRetry([&](){ block_blob_client.Upload(memory_stream); }, max_unexpected_write_error_retries, data_size); + LOG_TRACE(log, "Committed single block for blob `{}`", blob_path); + return; + } + } + + execWithRetry([this](){ next(); }, max_unexpected_write_error_retries); task_tracker->waitAll(); - auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); execWithRetry([&](){ block_blob_client.CommitBlockList(block_ids); }, max_unexpected_write_error_retries); LOG_TRACE(log, "Committed {} blocks for blob `{}`", block_ids.size(), blob_path); @@ -116,6 +130,8 @@ void WriteBufferFromAzureBlobStorage::allocateBuffer() size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), size); } + LOG_INFO(log, "allocateBuffer size = {}", size); + memory = Memory(size); WriteBuffer::set(memory.data(), memory.size()); } @@ -129,6 +145,8 @@ void WriteBufferFromAzureBlobStorage::writePart() if (data_size == 0) return; + LOG_TRACE(log, "writePart data size `{}`", data_size); + auto upload_worker = [&] () { auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index 3c06801a070..e7ee768876f 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -213,7 +213,7 @@ std::unique_ptr getAzureBlobContainerClient( std::unique_ptr getAzureBlobStorageSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context) { std::unique_ptr settings = std::make_unique(); - settings->max_single_part_upload_size = config.getUInt64(config_prefix + ".max_single_part_upload_size", 100 * 1024 * 1024); + settings->max_single_part_upload_size = config.getUInt64(config_prefix + ".max_single_part_upload_size", context->getSettings().azure_max_single_part_upload_size); settings->min_bytes_for_seek = config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024); settings->max_single_read_retries = config.getInt(config_prefix + ".max_single_read_retries", 3); settings->max_single_download_retries = config.getInt(config_prefix + ".max_single_download_retries", 3); diff --git a/tests/integration/test_merge_tree_azure_blob_storage/test.py b/tests/integration/test_merge_tree_azure_blob_storage/test.py index 55deb87a97e..cffab672bd1 100644 --- a/tests/integration/test_merge_tree_azure_blob_storage/test.py +++ b/tests/integration/test_merge_tree_azure_blob_storage/test.py @@ -37,6 +37,7 @@ def generate_cluster_def(port): devstoreaccount1 Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== 100000 + 100000 10 10 From f60b5f3cb7f1c32fd6f5d9837c4d27401dbe5da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 18:45:07 +0100 Subject: [PATCH 0244/1165] Add test for #60317 --- .../03014_analyzer_groupby_fuzz_60317.reference | 1 + .../0_stateless/03014_analyzer_groupby_fuzz_60317.sql | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference create mode 100644 tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql diff --git a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference new file mode 100644 index 00000000000..4972904f87d --- /dev/null +++ b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference @@ -0,0 +1 @@ +30 30 1970-01-01 diff --git a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql new file mode 100644 index 00000000000..2f54058526e --- /dev/null +++ b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql @@ -0,0 +1,9 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/60317 +SELECT + toNullable(materialize(_CAST(30, 'LowCardinality(UInt8)'))) as a, + _CAST(30, 'LowCardinality(UInt8)') as b, + makeDate(materialize(_CAST(30, 'LowCardinality(UInt8)')), 10, _CAST(30, 'Nullable(UInt8)')) as c +FROM system.one +GROUP BY + _CAST(30, 'Nullable(UInt8)') +SETTINGS allow_experimental_analyzer = 1; From e954f684823c98adfe5bc99af8de4c7a615a60a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 18:51:09 +0100 Subject: [PATCH 0245/1165] Add test for #60772 --- .../03015_analyzer_groupby_fuzz_60772.reference | 1 + .../0_stateless/03015_analyzer_groupby_fuzz_60772.sql | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference create mode 100644 tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql diff --git a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference new file mode 100644 index 00000000000..611407ecd90 --- /dev/null +++ b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference @@ -0,0 +1 @@ +%W 2018-01-02 22:33:44 1 diff --git a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql new file mode 100644 index 00000000000..c8b4eef50ff --- /dev/null +++ b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql @@ -0,0 +1,10 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/60772 +SELECT toFixedString(toFixedString(toFixedString(toFixedString(toFixedString(toFixedString('%W', 2), 2), 2),toLowCardinality(toLowCardinality(toNullable(2)))), 2), 2), + toFixedString(toFixedString('2018-01-02 22:33:44', 19), 19), + hasSubsequence(toNullable(materialize(toLowCardinality('garbage'))), 'gr') +GROUP BY + '2018-01-02 22:33:44', + toFixedString(toFixedString('2018-01-02 22:33:44', 19), 19), + 'gr', + '2018-01-02 22:33:44' +SETTINGS allow_experimental_analyzer = 1; From d56838080011e65ac2363862e8d6157bf2b35d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 19:03:35 +0100 Subject: [PATCH 0246/1165] Add test for #59796 --- .../0_stateless/03016_analyzer_groupby_fuzz_59796.reference | 1 + .../0_stateless/03016_analyzer_groupby_fuzz_59796.sql | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.reference create mode 100644 tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql diff --git a/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.reference b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.reference new file mode 100644 index 00000000000..6f7389b482b --- /dev/null +++ b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.reference @@ -0,0 +1 @@ +\0\0\0 key="v" \0 key="v" key="v" key="v" key="v" \0 key="v" diff --git a/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql new file mode 100644 index 00000000000..c00a75a631c --- /dev/null +++ b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql @@ -0,0 +1,6 @@ +SELECT + concat(concat(unhex('00'), concat(unhex('00'), concat(unhex(toFixedString('00', 2)), toFixedString(toFixedString(' key="v" ', 9), 9), concat(unhex('00'), toFixedString(' key="v" ', 9)), toFixedString(materialize(toLowCardinality(' key="v" ')), 9)), toFixedString(' key="v" ', 9)), toFixedString(' key="v" ', 9)), unhex('00'), ' key="v" ') AS haystack +GROUP BY + concat(unhex('00'), toFixedString(materialize(toFixedString(' key="v" ', 9)), 9), toFixedString(toFixedString('00', 2), toNullable(2)), toFixedString(toFixedString(toFixedString(' key="v" ', 9), 9), 9)), + concat(' key="v" ') +SETTINGS allow_experimental_analyzer = 1; From 73d601d231a8825643492b8895e6d1b43f04915f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 19:11:32 +0100 Subject: [PATCH 0247/1165] Add test for #61600 --- .../03017_analyzer_groupby_fuzz_61600.reference | 1 + .../03017_analyzer_groupby_fuzz_61600.sql | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference create mode 100644 tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql diff --git a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference new file mode 100644 index 00000000000..68acb650f8e --- /dev/null +++ b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference @@ -0,0 +1 @@ +38 \N diff --git a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql new file mode 100644 index 00000000000..44b27750c16 --- /dev/null +++ b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql @@ -0,0 +1,14 @@ +CREATE TABLE set_index_not__fuzz_0 (`name` String, `status` Enum8('alive' = 0, 'rip' = 1), INDEX idx_status status TYPE set(2) GRANULARITY 1) +ENGINE = MergeTree ORDER BY name +SETTINGS index_granularity = 8192; + +INSERT INTO set_index_not__fuzz_0 SELECT * from generateRandom() limit 1; + +SELECT + 38, + concat(position(concat(concat(position(concat(toUInt256(3)), 'ca', 2), 3),NULLIF(1, materialize(toLowCardinality(1)))), toLowCardinality(toNullable('ca'))), concat(NULLIF(1, 1), concat(3), toNullable(3))) +FROM set_index_not__fuzz_0 +GROUP BY + toNullable(3), + concat(concat(NULLIF(1, 1), toNullable(toNullable(3)))) +SETTINGS allow_experimental_analyzer = 1; From 221b74c77fd56fd184b770620af2c634f5227680 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 19 Mar 2024 19:44:15 +0100 Subject: [PATCH 0248/1165] Fix #ci_set_analyzer --- src/Analyzer/QueryNode.h | 5 +++++ src/Interpreters/ClusterProxy/executeQuery.cpp | 1 + src/Planner/PlannerJoinTree.cpp | 7 +++++++ src/Storages/StorageDistributed.cpp | 2 ++ src/Storages/buildQueryTreeForShard.cpp | 7 +++++++ 5 files changed, 22 insertions(+) diff --git a/src/Analyzer/QueryNode.h b/src/Analyzer/QueryNode.h index d8b8741afb2..6f9067908dd 100644 --- a/src/Analyzer/QueryNode.h +++ b/src/Analyzer/QueryNode.h @@ -99,6 +99,11 @@ public: return settings_changes; } + void clearSettingsChanges() + { + settings_changes.clear(); + } + /// Returns true if query node is subquery, false otherwise bool isSubquery() const { diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 07ef7aa6c96..2348081b56c 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -127,6 +127,7 @@ ContextMutablePtr updateSettingsForCluster(const Cluster & cluster, /// /// Here we don't try to analyze setting again. In case if query_info->additional_filter_ast is not empty, some filter was applied. /// It's just easier to add this filter for a source table. + LOG_DEBUG(&Poco::Logger::get("updateSettings"), "{}", additional_filter_ast != nullptr); if (additional_filter_ast) { Tuple tuple; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 8ca8f0f258b..e7afd7a1ea4 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1,5 +1,6 @@ #include +#include "Common/logger_useful.h" #include #include @@ -63,6 +64,7 @@ #include #include #include +#include namespace DB { @@ -523,6 +525,7 @@ FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage, return {}; auto const & storage_id = storage->getStorageID(); + LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", storage_id.getFullNameNotQuoted()); ASTPtr additional_filter_ast; for (const auto & additional_filter : additional_filters) @@ -543,6 +546,8 @@ FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage, } } + LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", additional_filter_ast != nullptr); + if (!additional_filter_ast) return {}; @@ -847,6 +852,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres const auto & table_expression_alias = table_expression->getOriginalAlias(); auto additional_filters_info = buildAdditionalFiltersIfNeeded(storage, table_expression_alias, table_expression_query_info, planner_context); + if (additional_filters_info.actions) + LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", additional_filters_info.actions->dumpDAG()); add_filter(additional_filters_info, "additional filter"); from_stage = storage->getQueryProcessingStage( diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index a9b909bc71e..675e5b1071a 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -836,6 +836,8 @@ void StorageDistributed::read( { Block header; + LOG_DEBUG(&Poco::Logger::get("read"), "{}", query_info.additional_filter_ast != nullptr); + SelectQueryInfo modified_query_info = query_info; if (local_context->getSettingsRef().allow_experimental_analyzer) diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index c87a1b216ca..ff88172e7a3 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -20,6 +20,7 @@ #include #include #include +#include "Analyzer/QueryNode.h" namespace DB { @@ -383,6 +384,12 @@ QueryTreeNodePtr buildQueryTreeForShard(const PlannerContextPtr & planner_contex createUniqueTableAliases(query_tree_to_modify, nullptr, planner_context->getQueryContext()); + // Get rid of the settings clause so we don't send them to remote. Thus newly non-important + // settings won't break any remote parser. It's also more reasonable since the query settings + // are written into the query context and will be sent by the query pipeline. + if (auto * query_node = query_tree_to_modify->as()) + query_node->clearSettingsChanges(); + return query_tree_to_modify; } From b561560c9968cacef84b62564e2dbf37182bcb6f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 19 Mar 2024 19:55:59 +0100 Subject: [PATCH 0249/1165] Remove it from remaining places --- src/Common/SensitiveDataMasker.cpp | 2 +- .../JoinToSubqueryTransformVisitor.cpp | 8 ++-- .../TranslateQualifiedNamesVisitor.cpp | 8 ++-- src/Parsers/ASTColumnsMatcher.cpp | 6 --- src/Parsers/ASTColumnsTransformers.cpp | 40 +++++++++---------- src/Parsers/ASTColumnsTransformers.h | 9 ++--- .../Formats/Impl/AvroRowOutputFormat.cpp | 2 +- 7 files changed, 33 insertions(+), 42 deletions(-) diff --git a/src/Common/SensitiveDataMasker.cpp b/src/Common/SensitiveDataMasker.cpp index 28eae6f451d..8c29b899841 100644 --- a/src/Common/SensitiveDataMasker.cpp +++ b/src/Common/SensitiveDataMasker.cpp @@ -60,7 +60,7 @@ public: , replacement(replacement_string) { if (!regexp.ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, "SensitiveDataMasker: cannot compile re2: {}, error: {}. " "Look at https://github.com/google/re2/wiki/Syntax for reference.", regexp_string_, regexp.error()); diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 209f8e68b8f..5cda4c982b4 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -205,16 +205,16 @@ private: has_asterisks = true; String pattern = columns_regexp_matcher->getPattern(); - auto regexp = std::make_shared(pattern, re2::RE2::Quiet); - if (!regexp->ok()) + re2::RE2 regexp(pattern, re2::RE2::Quiet); + if (!regexp.ok()) throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp.error()); for (auto & table_name : data.tables_order) data.addTableColumns( table_name, columns, - [&](const String & column_name) { return re2::RE2::PartialMatch(column_name, *regexp); }); + [&](const String & column_name) { return re2::RE2::PartialMatch(column_name, regexp); }); if (columns_regexp_matcher->transformers) { diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index e3690d0fcc0..03df7283992 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -278,17 +278,17 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt else if (const auto * asterisk_regexp_pattern = child->as()) { String pattern = asterisk_regexp_pattern->getPattern(); - auto regexp = std::make_shared(pattern, re2::RE2::Quiet); - if (!regexp->ok()) + re2::RE2 regexp(pattern, re2::RE2::Quiet); + if (!regexp.ok()) throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp.error()); bool first_table = true; for (const auto & table : tables_with_columns) { for (const auto & column : table.columns) { - if (re2::RE2::PartialMatch(column.name, *regexp) + if (re2::RE2::PartialMatch(column.name, regexp) && (first_table || !data.join_using_columns.contains(column.name))) { addIdentifier(columns, table.table, column.name); diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp index dac3661d482..518fd9b722d 100644 --- a/src/Parsers/ASTColumnsMatcher.cpp +++ b/src/Parsers/ASTColumnsMatcher.cpp @@ -4,17 +4,11 @@ #include #include #include -#include namespace DB { -namespace ErrorCodes -{ - extern const int CANNOT_COMPILE_REGEXP; -} - ASTPtr ASTColumnsRegexpMatcher::clone() const { auto clone = std::make_shared(*this); diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 34a1ae6e8e0..7f3fda9914c 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -12,6 +12,7 @@ namespace DB { + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -179,8 +180,8 @@ void ASTColumnsExceptTransformer::formatImpl(const FormatSettings & settings, Fo (*it)->formatImpl(settings, state, frame); } - if (!original_pattern.empty()) - settings.ostr << quoteString(original_pattern); + if (!pattern.empty()) + settings.ostr << quoteString(pattern); if (children.size() > 1) settings.ostr << ")"; @@ -202,8 +203,8 @@ void ASTColumnsExceptTransformer::appendColumnName(WriteBuffer & ostr) const (*it)->appendColumnName(ostr); } - if (!original_pattern.empty()) - writeQuotedString(original_pattern, ostr); + if (!pattern.empty()) + writeQuotedString(pattern, ostr); if (children.size() > 1) writeChar(')', ostr); @@ -212,8 +213,8 @@ void ASTColumnsExceptTransformer::appendColumnName(WriteBuffer & ostr) const void ASTColumnsExceptTransformer::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const { hash_state.update(is_strict); - hash_state.update(original_pattern.size()); - hash_state.update(original_pattern); + hash_state.update(pattern.size()); + hash_state.update(pattern); IAST::updateTreeHashImpl(hash_state, ignore_aliases); } @@ -221,7 +222,7 @@ void ASTColumnsExceptTransformer::updateTreeHashImpl(SipHash & hash_state, bool void ASTColumnsExceptTransformer::transform(ASTs & nodes) const { std::set expected_columns; - if (original_pattern.empty()) + if (pattern.empty()) { for (const auto & child : children) expected_columns.insert(child->as().name()); @@ -243,11 +244,13 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const } else { + auto regexp = getMatcher(); + for (auto * it = nodes.begin(); it != nodes.end();) { if (const auto * id = it->get()->as()) { - if (isColumnMatching(id->shortName())) + if (RE2::PartialMatch(id->shortName(), *regexp)) { it = nodes.erase(it); continue; @@ -268,23 +271,18 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const } } -void ASTColumnsExceptTransformer::setPattern(String pattern) +void ASTColumnsExceptTransformer::setPattern(String pattern_) { - original_pattern = std::move(pattern); - column_matcher = std::make_shared(original_pattern, RE2::Quiet); - if (!column_matcher->ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, "COLUMNS pattern {} cannot be compiled: {}", - original_pattern, column_matcher->error()); + pattern = std::move(pattern_); } -const std::shared_ptr & ASTColumnsExceptTransformer::getMatcher() const +std::shared_ptr ASTColumnsExceptTransformer::getMatcher() const { - return column_matcher; -} - -bool ASTColumnsExceptTransformer::isColumnMatching(const String & column_name) const -{ - return RE2::PartialMatch(column_name, *column_matcher); + auto regexp = std::make_shared(pattern, re2::RE2::Quiet); + if (!regexp->ok()) + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + return regexp; } void ASTColumnsReplaceTransformer::Replacement::formatImpl( diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index a2a138e13c9..2d13cd85819 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -2,6 +2,7 @@ #include + namespace re2 { class RE2; @@ -76,16 +77,14 @@ public: return clone; } void transform(ASTs & nodes) const override; - void setPattern(String pattern); - const std::shared_ptr & getMatcher() const; - bool isColumnMatching(const String & column_name) const; + void setPattern(String pattern_); + std::shared_ptr getMatcher() const; void appendColumnName(WriteBuffer & ostr) const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - std::shared_ptr column_matcher; - String original_pattern; + String pattern; }; class ASTColumnsReplaceTransformer : public IASTColumnsTransformer diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 07700942b57..a79a7d10c78 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -52,7 +52,7 @@ public: : string_to_string_regexp(settings_.avro.string_column_pattern) { if (!string_to_string_regexp.ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, "Avro: cannot compile re2: {}, error: {}. " + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, "Avro: cannot compile re2: {}, error: {}. " "Look at https://github.com/google/re2/wiki/Syntax for reference.", settings_.avro.string_column_pattern, string_to_string_regexp.error()); } From 1913466da4487784582df7301fe10464a80b2a4a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 19 Mar 2024 20:31:00 +0100 Subject: [PATCH 0250/1165] fix uaf of async_insert_queue --- src/Interpreters/AsynchronousInsertQueue.cpp | 15 ++++++++++++++- src/Interpreters/AsynchronousInsertQueue.h | 2 ++ src/Interpreters/Context.cpp | 8 +++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 9234d052d97..e82a902e03b 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -218,7 +218,7 @@ AsynchronousInsertQueue::AsynchronousInsertQueue(ContextPtr context_, size_t poo dump_by_first_update_threads.emplace_back([this, i] { processBatchDeadlines(i); }); } -AsynchronousInsertQueue::~AsynchronousInsertQueue() +void AsynchronousInsertQueue::flushAndShutdown() { try { @@ -257,6 +257,19 @@ AsynchronousInsertQueue::~AsynchronousInsertQueue() } } +AsynchronousInsertQueue::~AsynchronousInsertQueue() +{ + for (const auto & shard : queue_shards) + { + for (const auto & [first_update, elem] : shard.queue) + { + const auto & insert_query = elem.key.query->as(); + LOG_WARNING(log, "Has unprocessed async insert for {}.{}", + backQuoteIfNeed(insert_query.getDatabase()), backQuoteIfNeed(insert_query.getTable())); + } + } +} + void AsynchronousInsertQueue::scheduleDataProcessingJob( const InsertQuery & key, InsertDataPtr data, ContextPtr global_context, size_t shard_num) { diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 1a006cce85c..1a4678e3a7d 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -63,6 +63,8 @@ public: PushResult pushQueryWithBlock(ASTPtr query, Block block, ContextPtr query_context); size_t getPoolSize() const { return pool_size; } + void flushAndShutdown(); + private: struct InsertQuery diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index b1d2621e925..f388cf6b3de 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -566,7 +566,7 @@ struct ContextSharedPart : boost::noncopyable std::lock_guard lock(mutex); delete_async_insert_queue = std::move(async_insert_queue); } - delete_async_insert_queue.reset(); + delete_async_insert_queue->flushAndShutdown(); /// Stop periodic reloading of the configuration files. /// This must be done first because otherwise the reloading may pass a changed config @@ -590,6 +590,8 @@ struct ContextSharedPart : boost::noncopyable LOG_TRACE(log, "Shutting down database catalog"); DatabaseCatalog::shutdown(); + delete_async_insert_queue.reset(); + SHUTDOWN(log, "merges executor", merge_mutate_executor, wait()); SHUTDOWN(log, "fetches executor", fetch_executor, wait()); SHUTDOWN(log, "moves executor", moves_executor, wait()); @@ -4990,7 +4992,7 @@ PartUUIDsPtr Context::getIgnoredPartUUIDs() const AsynchronousInsertQueue * Context::tryGetAsynchronousInsertQueue() const { - std::lock_guard lock(mutex); + SharedLockGuard lock(shared->mutex); return shared->async_insert_queue.get(); } @@ -4998,7 +5000,7 @@ void Context::setAsynchronousInsertQueue(const std::shared_ptrmutex); if (std::chrono::milliseconds(settings.async_insert_poll_timeout_ms) == std::chrono::milliseconds::zero()) throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting async_insert_poll_timeout_ms can't be zero"); From 11d0b0a9fad7414cd5be6d3cccac80c1e5f66a91 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 Mar 2024 19:22:02 +0100 Subject: [PATCH 0251/1165] Forbid SimpleAggregateFunction in ORDER BY of MergeTree tables Like AggregateFunction is forbidden, but they are forbidden because they are not comparable. New setting (allow_suspicious_primary_key) had been added for backard compatiblity (turned OFF by default). Signed-off-by: Azat Khuzhin --- src/Core/Settings.h | 1 + src/Core/SettingsChangesHistory.h | 1 + .../MergeTree/registerStorageMergeTree.cpp | 24 ++++++++++++++++--- .../01410_nullable_key_and_index.sql | 2 +- ...order_by_SimpleAggregateFunction.reference | 0 ...03020_order_by_SimpleAggregateFunction.sql | 14 +++++++++++ 6 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/03020_order_by_SimpleAggregateFunction.reference create mode 100644 tests/queries/0_stateless/03020_order_by_SimpleAggregateFunction.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a3c5638d97f..7d53803e41f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -161,6 +161,7 @@ class IColumn; M(Bool, allow_suspicious_indices, false, "Reject primary/secondary indexes and sorting keys with identical expressions", 0) \ M(Bool, allow_suspicious_ttl_expressions, false, "Reject TTL expressions that don't depend on any of table's columns. It indicates a user error most of the time.", 0) \ M(Bool, allow_suspicious_variant_types, false, "In CREATE TABLE statement allows specifying Variant type with similar variant types (for example, with different numeric or date types). Enabling this setting may introduce some ambiguity when working with values with similar types.", 0) \ + M(Bool, allow_suspicious_primary_key, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)", 0) \ M(Bool, compile_expressions, false, "Compile some scalar functions and operators to native code.", 0) \ M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \ M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 072b9803682..2d8c173d850 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -95,6 +95,7 @@ static std::map sett {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."}, {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."}, + {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"}, {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"}, {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"}, {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"}, diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 2b24a56e994..e89547952d0 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -30,6 +31,7 @@ namespace ErrorCodes extern const int UNKNOWN_STORAGE; extern const int NO_REPLICA_NAME_GIVEN; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; + extern const int DATA_TYPE_CANNOT_BE_USED_IN_KEY; } @@ -110,6 +112,16 @@ static ColumnsDescription getColumnsDescriptionFromZookeeper(const String & raw_ return ColumnsDescription::parse(zookeeper->get(fs::path(zookeeper_path) / "columns", &columns_stat)); } +static void verifySortingKey(const KeyDescription & sorting_key) +{ + /// Aggregate functions already forbidden, but SimpleAggregateFunction are not + for (const auto & data_type : sorting_key.data_types) + { + if (dynamic_cast(data_type->getCustomName())) + throw Exception(ErrorCodes::DATA_TYPE_CANNOT_BE_USED_IN_KEY, "Column with type {} is not allowed in key expression", data_type->getCustomName()->getName()); + } +} + static StoragePtr create(const StorageFactory::Arguments & args) { @@ -148,6 +160,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) || (args.query.columns_list->indices && !args.query.columns_list->indices->children.empty()) || (args.query.columns_list->projections && !args.query.columns_list->projections->children.empty()); + const Settings & local_settings = args.getLocalContext()->getSettingsRef(); + String name_part = args.engine_name.substr(0, args.engine_name.size() - strlen("MergeTree")); bool replicated = startsWith(name_part, "Replicated"); @@ -293,7 +307,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) arg_idx, e.message(), verbose_help_message); } } - else if (args.mode <= LoadingStrictnessLevel::CREATE && !args.getLocalContext()->getSettingsRef().allow_deprecated_syntax_for_merge_tree) + else if (args.mode <= LoadingStrictnessLevel::CREATE && !local_settings.allow_deprecated_syntax_for_merge_tree) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "This syntax for *MergeTree engine is deprecated. " "Use extended storage definition syntax with ORDER BY/PRIMARY KEY clause. " @@ -532,7 +546,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (!args.storage_def->order_by) { - if (args.getLocalContext()->getSettingsRef().create_table_empty_primary_key_by_default) + if (local_settings.create_table_empty_primary_key_by_default) { args.storage_def->set(args.storage_def->order_by, makeASTFunction("tuple")); } @@ -553,6 +567,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) /// column if sorting key will be changed. metadata.sorting_key = KeyDescription::getSortingKeyFromAST( args.storage_def->order_by->ptr(), metadata.columns, context, merging_param_key_arg); + if (!local_settings.allow_suspicious_primary_key) + verifySortingKey(metadata.sorting_key); /// If primary key explicitly defined, than get it from AST if (args.storage_def->primary_key) @@ -577,7 +593,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (args.storage_def->sample_by) metadata.sampling_key = KeyDescription::getKeyFromAST(args.storage_def->sample_by->ptr(), metadata.columns, context); - bool allow_suspicious_ttl = LoadingStrictnessLevel::SECONDARY_CREATE <= args.mode || args.getLocalContext()->getSettingsRef().allow_suspicious_ttl_expressions; + bool allow_suspicious_ttl = LoadingStrictnessLevel::SECONDARY_CREATE <= args.mode || local_settings.allow_suspicious_ttl_expressions; if (args.storage_def->ttl_table) { @@ -665,6 +681,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) /// column if sorting key will be changed. metadata.sorting_key = KeyDescription::getSortingKeyFromAST(engine_args[arg_num], metadata.columns, context, merging_param_key_arg); + if (!local_settings.allow_suspicious_primary_key) + verifySortingKey(metadata.sorting_key); /// In old syntax primary_key always equals to sorting key. metadata.primary_key = KeyDescription::getKeyFromAST(engine_args[arg_num], metadata.columns, context); diff --git a/tests/queries/0_stateless/01410_nullable_key_and_index.sql b/tests/queries/0_stateless/01410_nullable_key_and_index.sql index 905d997d95c..7c28a7a6e70 100644 --- a/tests/queries/0_stateless/01410_nullable_key_and_index.sql +++ b/tests/queries/0_stateless/01410_nullable_key_and_index.sql @@ -73,5 +73,5 @@ CREATE TABLE invalid_lc_null (id LowCardinality(Nullable(String))) ENGINE = Merg CREATE TABLE invalid_array_null (id Array(Nullable(String))) ENGINE = MergeTree ORDER BY id; -- { serverError 44 } CREATE TABLE invalid_tuple_null (id Tuple(Nullable(String), UInt8)) ENGINE = MergeTree ORDER BY id; -- { serverError 44 } CREATE TABLE invalid_map_null (id Map(UInt8, Nullable(String))) ENGINE = MergeTree ORDER BY id; -- { serverError 44 } -CREATE TABLE invalid_simple_agg_state_null (id SimpleAggregateFunction(sum, Nullable(UInt64))) ENGINE = MergeTree ORDER BY id; -- { serverError 44 } +CREATE TABLE invalid_simple_agg_state_null (id SimpleAggregateFunction(sum, Nullable(UInt64))) ENGINE = MergeTree ORDER BY id; -- { serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY } -- AggregateFunctions are not comparable and cannot be used in key expressions. No need to test it. diff --git a/tests/queries/0_stateless/03020_order_by_SimpleAggregateFunction.reference b/tests/queries/0_stateless/03020_order_by_SimpleAggregateFunction.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03020_order_by_SimpleAggregateFunction.sql b/tests/queries/0_stateless/03020_order_by_SimpleAggregateFunction.sql new file mode 100644 index 00000000000..fd3418fb50e --- /dev/null +++ b/tests/queries/0_stateless/03020_order_by_SimpleAggregateFunction.sql @@ -0,0 +1,14 @@ +set allow_suspicious_primary_key = 0; + +create table data (key Int, value AggregateFunction(sum, UInt64)) engine=AggregatingMergeTree() order by (key, value); -- { serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY } +create table data (key Int, value SimpleAggregateFunction(sum, UInt64)) engine=AggregatingMergeTree() order by (key, value); -- { serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY } + +create table data (key Int, value AggregateFunction(sum, UInt64)) engine=AggregatingMergeTree() primary key value; -- { serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY } +create table data (key Int, value SimpleAggregateFunction(sum, UInt64)) engine=AggregatingMergeTree() primary key value; -- { serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY } + +create table data (key Int, value AggregateFunction(sum, UInt64)) engine=AggregatingMergeTree() primary key value order by (value, key); -- { serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY } +create table data (key Int, value SimpleAggregateFunction(sum, UInt64)) engine=AggregatingMergeTree() primary key value order by (value, key); -- { serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY } + +set allow_suspicious_primary_key = 1; + +create table data (key Int, value SimpleAggregateFunction(sum, UInt64)) engine=AggregatingMergeTree() primary key value order by (value, key); From 2f4b9b3a5b824ea255e1ea17c522f68d01cd0967 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 19 Mar 2024 16:08:20 +0100 Subject: [PATCH 0252/1165] Add few notes and a debug check. --- .../AbstractConfigurationComparison.cpp | 29 ++++++++++--------- .../Config/AbstractConfigurationComparison.h | 12 ++++---- src/Common/Config/ConfigHelper.h | 2 ++ .../ExternalDictionariesLoader.cpp | 2 +- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/Common/Config/AbstractConfigurationComparison.cpp b/src/Common/Config/AbstractConfigurationComparison.cpp index e983bd32f51..e241a540cc1 100644 --- a/src/Common/Config/AbstractConfigurationComparison.cpp +++ b/src/Common/Config/AbstractConfigurationComparison.cpp @@ -37,6 +37,12 @@ namespace { std::erase_if(left_subkeys, [&](const String & key) { return ignore_keys->contains(key); }); std::erase_if(right_subkeys, [&](const String & key) { return ignore_keys->contains(key); }); + +#if defined(ABORT_ON_LOGICAL_ERROR) + /// Compound `ignore_keys` are not yet implemented. + for (const auto & ignore_key : *ignore_keys) + chassert(ignore_key.find(".") == std::string_view::npos); +#endif } /// Check that the right configuration has the same set of subkeys as the left configuration. @@ -63,7 +69,7 @@ namespace if (!left_subkeys_set.contains(subkey)) return false; - if (!isSameConfiguration(left, concatKeyAndSubKey(left_key, subkey), right, concatKeyAndSubKey(right_key, subkey))) + if (!isSameConfigurationImpl(left, concatKeyAndSubKey(left_key, subkey), right, concatKeyAndSubKey(right_key, subkey), nullptr)) return false; } return true; @@ -82,6 +88,14 @@ bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const P return isSameConfiguration(left, key, right, key); } +bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set & ignore_keys) +{ + const auto * ignore_keys_ptr = !ignore_keys.empty() ? &ignore_keys : nullptr; + return isSameConfigurationImpl(left, left_key, right, right_key, ignore_keys_ptr); +} + bool isSameConfigurationWithMultipleKeys(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right, const String & root, const String & name) { if (&left == &right) @@ -99,17 +113,4 @@ bool isSameConfigurationWithMultipleKeys(const Poco::Util::AbstractConfiguration return true; } -bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, - const Poco::Util::AbstractConfiguration & right, const String & right_key) -{ - return isSameConfigurationImpl(left, left_key, right, right_key, /* ignore_keys= */ nullptr); -} - -bool isSameConfigurationIgnoringKeys(const Poco::Util::AbstractConfiguration & left, const String & left_key, - const Poco::Util::AbstractConfiguration & right, const String & right_key, - const std::unordered_set & ignore_keys) -{ - return isSameConfigurationImpl(left, left_key, right, right_key, !ignore_keys.empty() ? &ignore_keys : nullptr); -} - } diff --git a/src/Common/Config/AbstractConfigurationComparison.h b/src/Common/Config/AbstractConfigurationComparison.h index edaff8d5363..e8f65a4afcc 100644 --- a/src/Common/Config/AbstractConfigurationComparison.h +++ b/src/Common/Config/AbstractConfigurationComparison.h @@ -11,6 +11,8 @@ namespace Poco::Util namespace DB { /// Returns true if two configurations contains the same keys and values. + /// NOTE: These functions assume no configuration has items having both children and a value + /// (i.e. items like "value"). bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right); @@ -31,13 +33,11 @@ namespace DB const String & key); /// Returns true if specified subviews of the two configurations contains the same keys and values. + /// If `ignore_keys` is specified then the function skips those keys while comparing + /// (even if their values differ, they're considered to be the same.) bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, - const Poco::Util::AbstractConfiguration & right, const String & right_key); - - /// Returns true if specified subviews of the two configurations contains the same keys and values, but without checking specified keys. - bool isSameConfigurationIgnoringKeys(const Poco::Util::AbstractConfiguration & left, const String & left_key, - const Poco::Util::AbstractConfiguration & right, const String & right_key, - const std::unordered_set & ignore_keys); + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set & ignore_keys = {}); inline bool operator==(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right) { diff --git a/src/Common/Config/ConfigHelper.h b/src/Common/Config/ConfigHelper.h index 4f13bc4ad78..513438bd859 100644 --- a/src/Common/Config/ConfigHelper.h +++ b/src/Common/Config/ConfigHelper.h @@ -14,6 +14,8 @@ namespace DB::ConfigHelper { /// Clones a configuration. +/// NOTE: This function assumes the source configuration doesn't have items having both children and a value +/// (i.e. items like "value"). Poco::AutoPtr clone(const Poco::Util::AbstractConfiguration & src); /// The behavior is like `config.getBool(key, default_)`, diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 34ad240d089..f48ee61dab8 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -55,7 +55,7 @@ bool ExternalDictionariesLoader::doesConfigChangeRequiresReloadingObject(const P ignore_keys.insert("database"); } - return !isSameConfigurationIgnoringKeys(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); + return !isSameConfiguration(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); } void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const From 13ff65474ace15bcd7fc907242a5f7066dacec79 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 19 Mar 2024 21:09:03 +0000 Subject: [PATCH 0253/1165] impl --- src/Storages/System/StorageSystemReplicas.cpp | 6 +++++- .../02908_many_requests_to_system_replicas.reference | 1 + .../02908_many_requests_to_system_replicas.sh | 11 ++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index af8d67cbc21..35550de11cb 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -135,8 +135,12 @@ public: auto & [_, storage, promise, with_zk_fields] = req; - auto get_status_task = [this, storage, with_zk_fields, promise] () mutable + auto get_status_task = [this, storage, with_zk_fields, promise, thread_group = CurrentThread::getGroup()]() mutable { + SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachFromGroupIfNotDetached();); + if (thread_group) + CurrentThread::attachToGroupIfDetached(thread_group); + try { ReplicatedTableStatus status; diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference index f1ca07ef408..17c94686470 100644 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference @@ -3,3 +3,4 @@ Making 200 requests to system.replicas Query system.replicas while waiting for other concurrent requests to finish 0 900 +1 diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 2c57545e603..17e1d87963a 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -49,7 +49,7 @@ echo "Making $CONCURRENCY requests to system.replicas" for i in $(seq 1 $CONCURRENCY) do - curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT * FROM system.replicas WHERE database=currentDatabase() FORMAT Null;" 2>&1 || echo "query $i failed" & + curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT * FROM system.replicas WHERE database=currentDatabase() FORMAT Null SETTINGS log_comment='02908_many_requests';" &>/dev/null & done echo "Query system.replicas while waiting for other concurrent requests to finish" @@ -59,3 +59,12 @@ curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT sum(lost_part curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT sum(is_leader) FROM system.replicas WHERE database=currentDatabase();" 2>&1; wait; + +$CLICKHOUSE_CLIENT -nq " +SYSTEM FLUSH LOGS; + +-- without optimisation there are ~350K zk requests +SELECT sum(ProfileEvents['ZooKeeperTransactions']) < 30000 + FROM system.query_log + WHERE current_database=currentDatabase() AND log_comment='02908_many_requests'; +" From 82b18670a2def4a853fc612344a948b584f59a61 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 19 Mar 2024 22:31:26 +0100 Subject: [PATCH 0254/1165] Fix error --- src/Analyzer/ColumnTransformers.cpp | 6 +++--- src/Analyzer/ColumnTransformers.h | 13 ++++++------- src/Analyzer/MatcherNode.cpp | 11 +++++++---- src/Parsers/ASTColumnsTransformers.cpp | 3 +++ 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Analyzer/ColumnTransformers.cpp b/src/Analyzer/ColumnTransformers.cpp index 40e1e019d50..3a6b9e3b291 100644 --- a/src/Analyzer/ColumnTransformers.cpp +++ b/src/Analyzer/ColumnTransformers.cpp @@ -1,18 +1,18 @@ #include #include +#include #include -#include #include #include -#include #include #include #include + namespace DB { @@ -133,7 +133,7 @@ ExceptColumnTransformerNode::ExceptColumnTransformerNode(std::shared_ptr #include #include -#include + + +namespace re2 +{ + class RE2; +} namespace DB { @@ -102,8 +107,6 @@ enum class ApplyColumnTransformerType /// Get apply column transformer type name const char * toString(ApplyColumnTransformerType type); -class ApplyColumnTransformerNode; -using ApplyColumnTransformerNodePtr = std::shared_ptr; /// Apply column transformer class ApplyColumnTransformerNode final : public IColumnTransformerNode @@ -158,8 +161,6 @@ enum class ExceptColumnTransformerType const char * toString(ExceptColumnTransformerType type); -class ExceptColumnTransformerNode; -using ExceptColumnTransformerNodePtr = std::shared_ptr; /** Except column transformer. * Strict EXCEPT column transformer must use all column names during matched nodes transformation. @@ -230,8 +231,6 @@ private: static constexpr size_t children_size = 0; }; -class ReplaceColumnTransformerNode; -using ReplaceColumnTransformerNodePtr = std::shared_ptr; /** Replace column transformer. * Strict replace column transformer must use all replacements during matched nodes transformation. diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index d0efb0646b3..05861bcc039 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -99,10 +99,13 @@ MatcherNode::MatcherNode(MatcherNodeType matcher_type_, , qualified_identifier(qualified_identifier_) , columns_identifiers(columns_identifiers_) { - columns_matcher = std::make_shared(pattern_, re2::RE2::Quiet); - if (!columns_matcher->ok()) - throw DB::Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern_, columns_matcher->error()); + if (!pattern_.empty()) + { + columns_matcher = std::make_shared(pattern_, re2::RE2::Quiet); + if (!columns_matcher->ok()) + throw DB::Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern_, columns_matcher->error()); + } auto column_transformers_list_node = std::make_shared(); diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 7f3fda9914c..42c4c6ad88a 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -278,6 +278,9 @@ void ASTColumnsExceptTransformer::setPattern(String pattern_) std::shared_ptr ASTColumnsExceptTransformer::getMatcher() const { + if (pattern.empty()) + return {}; + auto regexp = std::make_shared(pattern, re2::RE2::Quiet); if (!regexp->ok()) throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, From 7275caf5bb3f0f555bbf2155c61822d5f6cdb0de Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 19 Mar 2024 22:40:00 +0100 Subject: [PATCH 0255/1165] Update Context.cpp --- src/Interpreters/Context.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index f388cf6b3de..b3f16a9ba52 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -566,7 +566,8 @@ struct ContextSharedPart : boost::noncopyable std::lock_guard lock(mutex); delete_async_insert_queue = std::move(async_insert_queue); } - delete_async_insert_queue->flushAndShutdown(); + if (delete_async_insert_queue) + delete_async_insert_queue->flushAndShutdown(); /// Stop periodic reloading of the configuration files. /// This must be done first because otherwise the reloading may pass a changed config From 49401629593e9be43a0094015e2565b34f9f2716 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Mar 2024 22:44:08 +0100 Subject: [PATCH 0256/1165] Add comments --- src/Interpreters/Cache/EvictionCandidates.cpp | 1 - src/Interpreters/Cache/FileCache.cpp | 15 ++++++++++++--- src/Interpreters/Cache/IFileCachePriority.h | 2 ++ src/Interpreters/Cache/LRUFileCachePriority.cpp | 6 ++++-- src/Interpreters/Cache/SLRUFileCachePriority.cpp | 3 --- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 997b770f953..7f77f14ee76 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -37,7 +37,6 @@ void EvictionCandidates::add(const FileSegmentMetadataPtr & candidate, LockedKey it->second.key_metadata = locked_key.getKeyMetadata(); it->second.candidates.push_back(candidate); - chassert(!candidate->isEvicting(lock)); candidate->setEvicting(true, &locked_key, &lock); ++candidates_size; } diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 1d271b58dcc..b51e402bff9 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -863,11 +863,20 @@ bool FileCache::tryReserve( { chassert(reached_size_limit || reached_elements_limit); - size_t hold_size = reached_size_limit + /// If we did not reach size limit (it means we reached only elements limit here) + /// then we need to make sure that this fact that we fit in cache by size + /// remains true after we release the lock and take it again. + /// For this purpose we create a HoldSpace holder which makes sure that the space is hold in the meantime. + /// We substract reserve_stat.stat.releasable_size from the hold space, + /// because it is the space that will be released, so we do not need to take it into account. + const size_t hold_size = reached_size_limit ? size > reserve_stat.stat.releasable_size ? size - reserve_stat.stat.releasable_size : 0 : size; - size_t hold_elements = reached_elements_limit ? 0 : 1; - auto hold_space = std::make_unique(hold_size, hold_elements, queue_iterator, *main_priority, cache_lock); + /// If we reached the elements limit - we will evict at least 1 element, + /// then we do not need to hold anything, otherwise (if we reached limit only by size) + /// we will also evict at least one element, so hold elements count is awlays zero here. + auto hold_space = std::make_unique( + hold_size, /* hold_elements */0, queue_iterator, *main_priority, cache_lock); cache_lock.unlock(); try diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index 8b1654f1454..c5d440df915 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -46,6 +46,8 @@ public: if (evicting_ && (!locked_key || !lock)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Setting evicting state to `true` can be done only under lock"); + + chassert(evicting.load() != evicting_); evicting.store(evicting_); } diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index e1004cc5245..461818678c1 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -468,8 +468,10 @@ void LRUFileCachePriority::holdImpl(size_t size, size_t elements, IteratorPtr, c if (!canFit(size, elements, lock)) { throw Exception(ErrorCodes::LOGICAL_ERROR, - "Cannot take space {}. Current state {}/{} in size, {}/{} in elements", - size, state->current_size, max_size, state->current_elements_num, max_elements); + "Cannot take space {} in size and {} in elements. " + "Current state {}/{} in size, {}/{} in elements", + size, elements, state->current_size, max_size, + state->current_elements_num, max_elements); } state->current_size += size; diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 6bc8f8461ea..99edfe7fa33 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -352,9 +352,6 @@ void SLRUFileCachePriority::SLRUIterator::assertValid() const void SLRUFileCachePriority::holdImpl(size_t size, size_t elements, IteratorPtr reservee, const CachePriorityGuard::Lock & lock) { - if (!canFit(size, elements, lock, reservee)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot hold space {}", size); - if (reservee) { const auto * slru_iterator = assert_cast(reservee.get()); From fd7c732c120585478cdd8300445bdba4f0394276 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Mar 2024 23:01:03 +0100 Subject: [PATCH 0257/1165] Add one more comment, better cache policy randomizer --- docker/test/stateful/run.sh | 14 +++++++------- src/Interpreters/Cache/FileCache.cpp | 2 +- src/Interpreters/Cache/IFileCachePriority.h | 9 +++++++++ src/Interpreters/Cache/SLRUFileCachePriority.cpp | 5 ----- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 18ac909df54..c2e9fdfe41d 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -25,7 +25,7 @@ azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml cache_policy="" -if [ $(( $(date +%-d) % 2 )) -eq 1 ]; then +if [ $(($RANDOM%2)) -eq 1 ]; then cache_policy="SLRU" else cache_policy="LRU" @@ -33,12 +33,12 @@ fi echo "Using cache policy: $cache_policy" -#if [ "$cache_policy" = "SLRU" ]; then -# sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ -# | sed "s|LRU|SLRU|" \ -# > /etc/clickhouse-server/config.d/storage_conf.xml.tmp -# mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml -#fi +if [ "$cache_policy" = "SLRU" ]; then + sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ + | sed "s|LRU|SLRU|" \ + > /etc/clickhouse-server/config.d/storage_conf.xml.tmp + mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml +fi if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then # It is not needed, we will explicitly create tables on s3. diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index b51e402bff9..1e03f6f9fd8 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -867,7 +867,7 @@ bool FileCache::tryReserve( /// then we need to make sure that this fact that we fit in cache by size /// remains true after we release the lock and take it again. /// For this purpose we create a HoldSpace holder which makes sure that the space is hold in the meantime. - /// We substract reserve_stat.stat.releasable_size from the hold space, + /// We subtract reserve_stat.stat.releasable_size from the hold space, /// because it is the space that will be released, so we do not need to take it into account. const size_t hold_size = reached_size_limit ? size > reserve_stat.stat.releasable_size ? size - reserve_stat.stat.releasable_size : 0 diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index c5d440df915..f275dfa485a 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -41,6 +41,15 @@ public: bool isEvicting(const CachePriorityGuard::Lock &) const { return evicting; } bool isEvicting(const LockedKey &) const { return evicting; } + /// This does not look good to have isEvicting with two options for locks, + /// but still it is valid as we do setEvicting always under both of them. + /// (Well, not always - only always for setting it to True, + /// but for False we have lower guarantees and allow a logical race, + /// physical race is not possible because the value is atomic). + /// We can avoid this ambiguity for isEvicting by introducing + /// a separate lock `EntryGuard::Lock`, it will make this part of code more coherent, + /// but it will introduce one more mutex while it is avoidable. + /// Introducing one more mutex just for coherency does not win the trade-off (isn't it?). void setEvicting(bool evicting_, const LockedKey * locked_key, const CachePriorityGuard::Lock * lock) const { if (evicting_ && (!locked_key || !lock)) diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 99edfe7fa33..07dd36e7655 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -10,11 +10,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - namespace { size_t getRatio(size_t total, double ratio) From 6eb488fe517df53aee24bc40dcc2f66a12052ffc Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Mar 2024 23:29:09 +0100 Subject: [PATCH 0258/1165] Ping CI From cfab620f49206bc3672e57813400857ff5541c7b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 20 Mar 2024 01:11:53 +0100 Subject: [PATCH 0259/1165] Fix bad code --- src/Analyzer/MatcherNode.cpp | 6 +++--- src/Analyzer/MatcherNode.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index 05861bcc039..48b60423688 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -53,7 +53,7 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, ColumnTransformersNod { } -MatcherNode::MatcherNode(const String & pattern_, ColumnTransformersNodes column_transformers_) +MatcherNode::MatcherNode(String pattern_, ColumnTransformersNodes column_transformers_) : MatcherNode(MatcherNodeType::COLUMNS_REGEXP, {} /*qualified_identifier*/, {} /*columns_identifiers*/, @@ -62,7 +62,7 @@ MatcherNode::MatcherNode(const String & pattern_, ColumnTransformersNodes column { } -MatcherNode::MatcherNode(Identifier qualified_identifier_, const String & pattern_, ColumnTransformersNodes column_transformers_) +MatcherNode::MatcherNode(Identifier qualified_identifier_, String pattern_, ColumnTransformersNodes column_transformers_) : MatcherNode(MatcherNodeType::COLUMNS_REGEXP, std::move(qualified_identifier_), {} /*columns_identifiers*/, @@ -92,7 +92,7 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, Identifiers columns_i MatcherNode::MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - const String & pattern_, + String pattern_, ColumnTransformersNodes column_transformers_) : IQueryTreeNode(children_size) , matcher_type(matcher_type_) diff --git a/src/Analyzer/MatcherNode.h b/src/Analyzer/MatcherNode.h index 2217fde6eff..9f6b7c6ce3d 100644 --- a/src/Analyzer/MatcherNode.h +++ b/src/Analyzer/MatcherNode.h @@ -59,10 +59,10 @@ public: explicit MatcherNode(Identifier qualified_identifier_, ColumnTransformersNodes column_transformers_ = {}); /// Variant unqualified COLUMNS('regexp') - explicit MatcherNode(const String & pattern_, ColumnTransformersNodes column_transformers_ = {}); + explicit MatcherNode(String pattern_, ColumnTransformersNodes column_transformers_ = {}); /// Variant qualified COLUMNS('regexp') - explicit MatcherNode(Identifier qualified_identifier_, const String & pattern_, ColumnTransformersNodes column_transformers_ = {}); + explicit MatcherNode(Identifier qualified_identifier_, String pattern_, ColumnTransformersNodes column_transformers_ = {}); /// Variant unqualified COLUMNS(column_name_1, ...) explicit MatcherNode(Identifiers columns_identifiers_, ColumnTransformersNodes column_transformers_ = {}); @@ -147,7 +147,7 @@ private: explicit MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - const String & pattern_, + String pattern_, ColumnTransformersNodes column_transformers_); MatcherNodeType matcher_type; From c63a5566dc8f54ed28f07ddf7097bbfeb7f8b0a7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 20 Mar 2024 01:25:46 +0100 Subject: [PATCH 0260/1165] Add a test --- .../03014_invalid_utf8_client.expect | 27 +++++++++++++++++++ .../03014_invalid_utf8_client.reference | 0 2 files changed, 27 insertions(+) create mode 100755 tests/queries/0_stateless/03014_invalid_utf8_client.expect create mode 100644 tests/queries/0_stateless/03014_invalid_utf8_client.reference diff --git a/tests/queries/0_stateless/03014_invalid_utf8_client.expect b/tests/queries/0_stateless/03014_invalid_utf8_client.expect new file mode 100755 index 00000000000..13cbfe408d4 --- /dev/null +++ b/tests/queries/0_stateless/03014_invalid_utf8_client.expect @@ -0,0 +1,27 @@ +#!/usr/bin/expect -f + +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history + +log_user 0 +set timeout 60 +match_max 100000 + +expect_after { + # Do not ignore eof from expect + -i $any_spawn_id eof { exp_continue } + # A default timeout action is to do nothing, change it to fail + -i $any_spawn_id timeout { exit 1 } +} + +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" +expect ":) " + +send -- "SELECT \x99\r" + +expect "Syntax error" + +send "exit\r" +expect eof diff --git a/tests/queries/0_stateless/03014_invalid_utf8_client.reference b/tests/queries/0_stateless/03014_invalid_utf8_client.reference new file mode 100644 index 00000000000..e69de29bb2d From a29d4f5db2aa6554ece3d1dbcef98928d899f6f4 Mon Sep 17 00:00:00 2001 From: liuneng <1398775315@qq.com> Date: Wed, 20 Mar 2024 10:24:34 +0800 Subject: [PATCH 0261/1165] fix bug --- src/Analyzer/Passes/ConvertInToEqualPass.cpp | 3 +-- src/Core/SettingsChangesHistory.h | 1 + tests/performance/function_in.xml | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp index 26db3a4f4aa..e89300cd967 100644 --- a/src/Analyzer/Passes/ConvertInToEqualPass.cpp +++ b/src/Analyzer/Passes/ConvertInToEqualPass.cpp @@ -36,11 +36,10 @@ public: return ; if (constant_node->getValue().getType() == Field::Types::Which::Tuple) return; - auto const_value = std::make_shared(constant_node->getValue()); auto equal_resolver = createInternalFunctionEqualOverloadResolver(); auto equal = std::make_shared("equals"); - QueryTreeNodes arguments{column_node->clone(), const_value}; + QueryTreeNodes arguments{column_node->clone(), constant_node->clone()}; equal->getArguments().getNodes() = std::move(arguments); equal->resolveAsFunction(equal_resolver); node = equal; diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 8e2b2915c2a..886dce72176 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -96,6 +96,7 @@ static std::map sett {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"}, {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"}, + {"optimize_in_to_equal", false, true, "Convert in single value to equal"}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, diff --git a/tests/performance/function_in.xml b/tests/performance/function_in.xml index 16e6bb91469..af4f8737ba7 100644 --- a/tests/performance/function_in.xml +++ b/tests/performance/function_in.xml @@ -21,10 +21,8 @@ ENGINE = Memory insert into t_nullable select ['aaaaaa','bbaaaa','ccaaaa','ddaaaa'][number % 101 + 1], ['aa','bb','cc','dd'][number % 100 + 1], ['aa','bb','cc','dd'][number % 102 + 1], number%10+1, number%10+2, number%10+3, number%10+4,number%10+5, number%6000+1, number%5000+2 from numbers_mt(30000000) - select * from t_nullable where key_string1 in ('aaaaaa') format Null select * from t_nullable where key_string1 in ('aaaaaa') format Null SETTINGS allow_experimental_analyzer=1 - select * from t_nullable where key_string2 in ('3') format Null select * from t_nullable where key_string2 in ('3') format Null SETTINGS allow_experimental_analyzer=1 drop table if exists t_nullable - \ No newline at end of file + From 862abf2f6efecf8a5acf6bf49f681b0d696c3db9 Mon Sep 17 00:00:00 2001 From: liuneng <1398775315@qq.com> Date: Wed, 20 Mar 2024 10:40:22 +0800 Subject: [PATCH 0262/1165] remove config --- src/Analyzer/Passes/ConvertInToEqualPass.cpp | 2 -- src/Core/Settings.h | 1 - src/Core/SettingsChangesHistory.h | 1 - 3 files changed, 4 deletions(-) diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp index e89300cd967..ce2cec9f9ed 100644 --- a/src/Analyzer/Passes/ConvertInToEqualPass.cpp +++ b/src/Analyzer/Passes/ConvertInToEqualPass.cpp @@ -24,8 +24,6 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_in_to_equal) - return; auto * func_node = node->as(); if (!func_node || func_node->getFunctionName() != "in" || func_node->getArguments().getNodes().size() != 2) return ; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 19655d6493d..e6adb00137f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -593,7 +593,6 @@ class IColumn; M(Bool, optimize_functions_to_subcolumns, false, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ M(Bool, optimize_using_constraints, false, "Use constraints for query optimization", 0) \ M(Bool, optimize_substitute_columns, false, "Use constraints for column substitution", 0) \ - M(Bool, optimize_in_to_equal, true, "Convert the constant in to equal", 0) \ M(Bool, optimize_append_index, false, "Use constraints in order to append index condition (indexHint)", 0) \ M(Bool, optimize_time_filter_with_preimage, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')", 0) \ M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 886dce72176..8e2b2915c2a 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -96,7 +96,6 @@ static std::map sett {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"}, {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"}, - {"optimize_in_to_equal", false, true, "Convert in single value to equal"}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, From 45bbef689a053489a141bd8e2e20dc422c1d11d2 Mon Sep 17 00:00:00 2001 From: liuneng <1398775315@qq.com> Date: Wed, 20 Mar 2024 12:29:48 +0800 Subject: [PATCH 0263/1165] fix error for in Null --- src/Analyzer/Passes/ConvertInToEqualPass.cpp | 7 +++++-- .../0_stateless/03013_optimize_in_to_equal.reference | 6 ++++++ tests/queries/0_stateless/03013_optimize_in_to_equal.sql | 4 +++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp index ce2cec9f9ed..45b8a91f7ad 100644 --- a/src/Analyzer/Passes/ConvertInToEqualPass.cpp +++ b/src/Analyzer/Passes/ConvertInToEqualPass.cpp @@ -32,9 +32,12 @@ public: auto * constant_node = args[1]->as(); if (!column_node || !constant_node) return ; + // IN multiple values is not supported if (constant_node->getValue().getType() == Field::Types::Which::Tuple) - return; - + return ; + // x IN null not equivalent to x = null + if (constant_node->hasSourceExpression() || constant_node->getValue().isNull()) + return ; auto equal_resolver = createInternalFunctionEqualOverloadResolver(); auto equal = std::make_shared("equals"); QueryTreeNodes arguments{column_node->clone(), constant_node->clone()}; diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.reference b/tests/queries/0_stateless/03013_optimize_in_to_equal.reference index f55a7b76eaf..9ef4de0f607 100644 --- a/tests/queries/0_stateless/03013_optimize_in_to_equal.reference +++ b/tests/queries/0_stateless/03013_optimize_in_to_equal.reference @@ -1,5 +1,11 @@ a 1 ------------------- +0 +0 +0 +0 +0 +------------------- QUERY id: 0 PROJECTION COLUMNS x String diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.sql b/tests/queries/0_stateless/03013_optimize_in_to_equal.sql index 9d80825ceb4..e4dba3a0aa2 100644 --- a/tests/queries/0_stateless/03013_optimize_in_to_equal.sql +++ b/tests/queries/0_stateless/03013_optimize_in_to_equal.sql @@ -5,6 +5,8 @@ INSERT INTO test VALUES ('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5); select * from test where x in ('a') SETTINGS allow_experimental_analyzer = 1; select '-------------------'; +select x in Null from test; +select '-------------------'; explain query tree select * from test where x in ('a') SETTINGS allow_experimental_analyzer = 1; select '-------------------'; -explain query tree select * from test where x in ('a','b') SETTINGS allow_experimental_analyzer = 1; \ No newline at end of file +explain query tree select * from test where x in ('a','b') SETTINGS allow_experimental_analyzer = 1; From 96e90438e0a1f47fc706792492cf86fe093d0b26 Mon Sep 17 00:00:00 2001 From: nemonlou Date: Wed, 20 Mar 2024 14:25:16 +0800 Subject: [PATCH 0264/1165] fix clang-tiny --- src/Processors/Transforms/MergeJoinTransform.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index b63598483ef..62361bef5e2 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -305,7 +305,7 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( left_to_right_key_remap[left_idx] = right_idx; } - auto smjPtr = typeid_cast(table_join.get()); + const auto *smjPtr = typeid_cast(table_join.get()); if (smjPtr) { null_direction_hint = smjPtr->getNullDirection(); From e7aa23d33236443f4fbe36e695a47c6ff4e857d6 Mon Sep 17 00:00:00 2001 From: YenchangChan Date: Mon, 18 Mar 2024 20:14:46 +0800 Subject: [PATCH 0265/1165] Remove duplicate code in copyS3File.cpp --- src/IO/S3/copyS3File.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 51518df268c..b780c1fc08f 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -194,11 +194,6 @@ namespace auto outcome = client_ptr->CompleteMultipartUpload(request); - if (blob_storage_log) - blob_storage_log->addEvent(BlobStorageLogElement::EventType::MultiPartUploadComplete, - dest_bucket, dest_key, /* local_path_ */ {}, /* data_size */ 0, - outcome.IsSuccess() ? nullptr : &outcome.GetError()); - if (blob_storage_log) blob_storage_log->addEvent(BlobStorageLogElement::EventType::MultiPartUploadComplete, dest_bucket, dest_key, /* local_path_ */ {}, /* data_size */ 0, From d2d5f3b323c3e8b7a4a9823ea14b48b516030907 Mon Sep 17 00:00:00 2001 From: liuneng <1398775315@qq.com> Date: Wed, 20 Mar 2024 17:29:35 +0800 Subject: [PATCH 0266/1165] add not in --- src/Analyzer/Passes/ConvertInToEqualPass.cpp | 38 ++++++++++++------- src/Analyzer/Passes/ConvertInToEqualPass.h | 8 +++- src/Functions/CMakeLists.txt | 2 + src/Functions/equals.cpp | 5 +++ src/Functions/equals.h | 11 ++++++ src/Functions/notEquals.cpp | 5 +++ src/Functions/notEquals.h | 11 ++++++ .../03013_optimize_in_to_equal.reference | 27 ++++++++++++- .../03013_optimize_in_to_equal.sql | 10 +++-- 9 files changed, 98 insertions(+), 19 deletions(-) create mode 100644 src/Functions/equals.h create mode 100644 src/Functions/notEquals.h diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp index 45b8a91f7ad..fbac09de7aa 100644 --- a/src/Analyzer/Passes/ConvertInToEqualPass.cpp +++ b/src/Analyzer/Passes/ConvertInToEqualPass.cpp @@ -3,29 +3,28 @@ #include #include #include -#include -#include +#include +#include namespace DB { -using FunctionEquals = FunctionComparison; - class ConvertInToEqualPassVisitor : public InDepthQueryTreeVisitorWithContext { public: using Base = InDepthQueryTreeVisitorWithContext; using Base::Base; - FunctionOverloadResolverPtr createInternalFunctionEqualOverloadResolver() - { - return std::make_unique(std::make_shared(getContext()->getSettings().decimal_check_overflow)); - } - void enterImpl(QueryTreeNodePtr & node) { + static const std::unordered_map MAPPING = { + {"in", "equals"}, + {"notIn", "notEquals"} + }; auto * func_node = node->as(); - if (!func_node || func_node->getFunctionName() != "in" || func_node->getArguments().getNodes().size() != 2) + if (!func_node + || !MAPPING.contains(func_node->getFunctionName()) + || func_node->getArguments().getNodes().size() != 2) return ; auto args = func_node->getArguments().getNodes(); auto * column_node = args[0]->as(); @@ -38,13 +37,26 @@ public: // x IN null not equivalent to x = null if (constant_node->hasSourceExpression() || constant_node->getValue().isNull()) return ; - auto equal_resolver = createInternalFunctionEqualOverloadResolver(); - auto equal = std::make_shared("equals"); + auto result_func_name = MAPPING.at(func_node->getFunctionName()); + auto equal = std::make_shared(result_func_name); QueryTreeNodes arguments{column_node->clone(), constant_node->clone()}; equal->getArguments().getNodes() = std::move(arguments); - equal->resolveAsFunction(equal_resolver); + FunctionOverloadResolverPtr resolver; + bool decimal_check_overflow = getContext()->getSettingsRef().decimal_check_overflow; + if (result_func_name == "equals") + { + resolver = createInternalFunctionEqualOverloadResolver(decimal_check_overflow); + } + else + { + resolver = createInternalFunctionNotEqualOverloadResolver(decimal_check_overflow); + } + equal->resolveAsFunction(resolver); node = equal; } +private: + FunctionOverloadResolverPtr equal_resolver; + FunctionOverloadResolverPtr not_equal_resolver; }; void ConvertInToEqualPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.h b/src/Analyzer/Passes/ConvertInToEqualPass.h index 7de75e94f04..bd4f8607c88 100644 --- a/src/Analyzer/Passes/ConvertInToEqualPass.h +++ b/src/Analyzer/Passes/ConvertInToEqualPass.h @@ -5,9 +5,15 @@ namespace DB { /** Optimize `in` to `equals` if possible. - * + * 1. convert in single value to equal * Example: SELECT * from test where x IN (1); * Result: SELECT * from test where x = 1; + * + * 2. convert not in single value to notEqual + * Example: SELECT * from test where x NOT IN (1); + * Result: SELECT * from test where x != 1; + * + * If value is null or tuple, do not convert. */ class ConvertInToEqualPass final : public IQueryTreePass { diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index ac3e3671ae0..399394011b5 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -14,6 +14,8 @@ extract_into_parent_list(clickhouse_functions_sources dbms_sources multiMatchAny.cpp checkHyperscanRegexp.cpp array/has.cpp + equals.cpp + notEquals.cpp CastOverloadResolver.cpp ) extract_into_parent_list(clickhouse_functions_headers dbms_headers diff --git a/src/Functions/equals.cpp b/src/Functions/equals.cpp index 5c59daf0537..512abaa6fc7 100644 --- a/src/Functions/equals.cpp +++ b/src/Functions/equals.cpp @@ -13,6 +13,11 @@ REGISTER_FUNCTION(Equals) factory.registerFunction(); } +FunctionOverloadResolverPtr createInternalFunctionEqualOverloadResolver(bool decimal_check_overflow) +{ + return std::make_unique(std::make_shared(decimal_check_overflow)); +} + template <> ColumnPtr FunctionComparison::executeTupleImpl( const ColumnsWithTypeAndName & x, const ColumnsWithTypeAndName & y, size_t tuple_size, size_t input_rows_count) const diff --git a/src/Functions/equals.h b/src/Functions/equals.h new file mode 100644 index 00000000000..855cba4db3e --- /dev/null +++ b/src/Functions/equals.h @@ -0,0 +1,11 @@ +#pragma once +#include + +namespace DB +{ + +class IFunctionOverloadResolver; +using FunctionOverloadResolverPtr = std::shared_ptr; + +FunctionOverloadResolverPtr createInternalFunctionEqualOverloadResolver(bool decimal_check_overflow); +} diff --git a/src/Functions/notEquals.cpp b/src/Functions/notEquals.cpp index 3a63db46711..744a0997d95 100644 --- a/src/Functions/notEquals.cpp +++ b/src/Functions/notEquals.cpp @@ -12,6 +12,11 @@ REGISTER_FUNCTION(NotEquals) factory.registerFunction(); } +FunctionOverloadResolverPtr createInternalFunctionNotEqualOverloadResolver(bool decimal_check_overflow) +{ + return std::make_unique(std::make_shared(decimal_check_overflow)); +} + template <> ColumnPtr FunctionComparison::executeTupleImpl( const ColumnsWithTypeAndName & x, const ColumnsWithTypeAndName & y, size_t tuple_size, size_t input_rows_count) const diff --git a/src/Functions/notEquals.h b/src/Functions/notEquals.h new file mode 100644 index 00000000000..961889d68d7 --- /dev/null +++ b/src/Functions/notEquals.h @@ -0,0 +1,11 @@ +#pragma once +#include + +namespace DB +{ + +class IFunctionOverloadResolver; +using FunctionOverloadResolverPtr = std::shared_ptr; + +FunctionOverloadResolverPtr createInternalFunctionNotEqualOverloadResolver(bool decimal_check_overflow); +} diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.reference b/tests/queries/0_stateless/03013_optimize_in_to_equal.reference index 9ef4de0f607..dcaa14abfb1 100644 --- a/tests/queries/0_stateless/03013_optimize_in_to_equal.reference +++ b/tests/queries/0_stateless/03013_optimize_in_to_equal.reference @@ -3,8 +3,6 @@ a 1 0 0 0 -0 -0 ------------------- QUERY id: 0 PROJECTION COLUMNS @@ -41,3 +39,28 @@ QUERY id: 0 COLUMN id: 7, column_name: x, result_type: String, source_id: 3 CONSTANT id: 8, constant_value: Tuple_(\'a\', \'b\'), constant_value_type: Tuple(String, String) SETTINGS allow_experimental_analyzer=1 +------------------- +b 2 +c 3 +------------------- +QUERY id: 0 + PROJECTION COLUMNS + x String + y Int32 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: x, result_type: String, source_id: 3 + COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.test + WHERE + FUNCTION id: 5, function_name: notEquals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + COLUMN id: 7, column_name: x, result_type: String, source_id: 3 + CONSTANT id: 8, constant_value: \'a\', constant_value_type: String + SETTINGS allow_experimental_analyzer=1 +------------------- +a 1 +b 2 +c 3 diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.sql b/tests/queries/0_stateless/03013_optimize_in_to_equal.sql index e4dba3a0aa2..be4dc2275a9 100644 --- a/tests/queries/0_stateless/03013_optimize_in_to_equal.sql +++ b/tests/queries/0_stateless/03013_optimize_in_to_equal.sql @@ -1,8 +1,6 @@ DROP TABLE IF EXISTS test; CREATE TABLE test (x String, y Int32) ENGINE = MergeTree() ORDER BY x; - -INSERT INTO test VALUES ('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5); - +INSERT INTO test VALUES ('a', 1), ('b', 2), ('c', 3); select * from test where x in ('a') SETTINGS allow_experimental_analyzer = 1; select '-------------------'; select x in Null from test; @@ -10,3 +8,9 @@ select '-------------------'; explain query tree select * from test where x in ('a') SETTINGS allow_experimental_analyzer = 1; select '-------------------'; explain query tree select * from test where x in ('a','b') SETTINGS allow_experimental_analyzer = 1; +select '-------------------'; +select * from test where x not in ('a') SETTINGS allow_experimental_analyzer = 1; +select '-------------------'; +explain query tree select * from test where x not in ('a') SETTINGS allow_experimental_analyzer = 1; +select '-------------------'; +select * from test where x not in (NULL) SETTINGS allow_experimental_analyzer = 1; From 10bf164aa732adc68d914eb3911f1c94aec77fcf Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 20 Mar 2024 10:47:15 +0100 Subject: [PATCH 0267/1165] Cleanup #ci_set_analyzer --- .../ClusterProxy/executeQuery.cpp | 1 - src/Planner/PlannerJoinTree.cpp | 7 ------- src/Storages/StorageDistributed.cpp | 2 -- src/Storages/buildQueryTreeForShard.cpp | 20 +++++++++---------- .../01455_opentelemetry_distributed.reference | 4 ++-- ..._parallel_replicas_join_subquery.reference | 2 +- ...02771_parallel_replicas_analyzer.reference | 2 +- 7 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 2348081b56c..07ef7aa6c96 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -127,7 +127,6 @@ ContextMutablePtr updateSettingsForCluster(const Cluster & cluster, /// /// Here we don't try to analyze setting again. In case if query_info->additional_filter_ast is not empty, some filter was applied. /// It's just easier to add this filter for a source table. - LOG_DEBUG(&Poco::Logger::get("updateSettings"), "{}", additional_filter_ast != nullptr); if (additional_filter_ast) { Tuple tuple; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index e7afd7a1ea4..8ca8f0f258b 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1,6 +1,5 @@ #include -#include "Common/logger_useful.h" #include #include @@ -64,7 +63,6 @@ #include #include #include -#include namespace DB { @@ -525,7 +523,6 @@ FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage, return {}; auto const & storage_id = storage->getStorageID(); - LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", storage_id.getFullNameNotQuoted()); ASTPtr additional_filter_ast; for (const auto & additional_filter : additional_filters) @@ -546,8 +543,6 @@ FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage, } } - LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", additional_filter_ast != nullptr); - if (!additional_filter_ast) return {}; @@ -852,8 +847,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres const auto & table_expression_alias = table_expression->getOriginalAlias(); auto additional_filters_info = buildAdditionalFiltersIfNeeded(storage, table_expression_alias, table_expression_query_info, planner_context); - if (additional_filters_info.actions) - LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", additional_filters_info.actions->dumpDAG()); add_filter(additional_filters_info, "additional filter"); from_stage = storage->getQueryProcessingStage( diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 675e5b1071a..a9b909bc71e 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -836,8 +836,6 @@ void StorageDistributed::read( { Block header; - LOG_DEBUG(&Poco::Logger::get("read"), "{}", query_info.additional_filter_ast != nullptr); - SelectQueryInfo modified_query_info = query_info; if (local_context->getSettingsRef().allow_experimental_analyzer) diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index ff88172e7a3..5015e89dff7 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -1,26 +1,26 @@ #include -#include #include +#include #include -#include #include -#include +#include #include +#include +#include #include #include #include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include "Analyzer/QueryNode.h" namespace DB { diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index b04a3a5ea82..2920b387aa2 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -1,9 +1,9 @@ ===http=== {"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} -{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1` SETTINGS allow_experimental_analyzer = 1","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} -{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1` SETTINGS allow_experimental_analyzer = 1","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"total spans":"3","unique spans":"3","unique non-zero parent spans":"3"} {"initial query spans with proper parent":"2"} diff --git a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference index 028cc744170..451f0d6d485 100644 --- a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference +++ b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference @@ -31,7 +31,7 @@ 29 2j&S)ba?XG QuQj 17163829389637435056 3 UlI+1 14144472852965836438 =============== QUERIES EXECUTED BY PARALLEL INNER QUERY ALONE =============== -0 3 SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\') SETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer = 1 +0 3 SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\') 0 3 SELECT `key`, `value1`, `value2`, toUInt64(min(`time`)) AS `start_ts` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2` ORDER BY `key` ASC, `value1` ASC, `value2` ASC LIMIT 10 1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=0; 1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=1; diff --git a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference index 3b8a394a522..5bf3520ccdb 100644 --- a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference +++ b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference @@ -9,4 +9,4 @@ 7885388429666205427 8124171311239967992 1 1 -- Simple query with analyzer and pure parallel replicas\nSELECT number\nFROM join_inner_table__fuzz_146_replicated\n SETTINGS\n allow_experimental_analyzer = 1,\n max_parallel_replicas = 2,\n cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\',\n allow_experimental_parallel_reading_from_replicas = 1; -0 2 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1` SETTINGS allow_experimental_analyzer = 1, max_parallel_replicas = 2, cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\', allow_experimental_parallel_reading_from_replicas = 1 +0 2 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1` From 2c0e266cfe13b48f9cb149683a5405a6b2fa41e1 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 20 Mar 2024 10:58:46 +0100 Subject: [PATCH 0268/1165] Addressed review comments --- src/Common/BufferAllocationPolicy.cpp | 2 +- src/Common/BufferAllocationPolicy.h | 3 +- .../IO/WriteBufferFromAzureBlobStorage.cpp | 53 +++++++++++-------- src/IO/WriteBufferFromS3.cpp | 32 +++++------ src/IO/WriteBufferFromS3.h | 1 - 5 files changed, 47 insertions(+), 44 deletions(-) diff --git a/src/Common/BufferAllocationPolicy.cpp b/src/Common/BufferAllocationPolicy.cpp index 14d009008b9..359da0f8313 100644 --- a/src/Common/BufferAllocationPolicy.cpp +++ b/src/Common/BufferAllocationPolicy.cpp @@ -73,7 +73,7 @@ public: if (1 == buffer_number) { - current_size = first_size; + current_size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), first_size); return; } diff --git a/src/Common/BufferAllocationPolicy.h b/src/Common/BufferAllocationPolicy.h index dd04139e694..4ac20f1605b 100644 --- a/src/Common/BufferAllocationPolicy.h +++ b/src/Common/BufferAllocationPolicy.h @@ -9,6 +9,7 @@ namespace DB { +/// Buffer number starts with 0 class IBufferAllocationPolicy { public: @@ -20,7 +21,7 @@ public: size_t max_size = 5ULL * 1024 * 1024 * 1024; size_t multiply_factor = 2; size_t multiply_parts_count_threshold = 500; - size_t max_single_size = 32 * 1024 * 1024; + size_t max_single_size = 32 * 1024 * 1024; /// Max size for a single buffer/block }; diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index f966e431506..1ef54272295 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -18,6 +18,25 @@ namespace ProfileEvents namespace DB { +struct WriteBufferFromAzureBlobStorage::PartData +{ + Memory<> memory; + size_t data_size = 0; +}; + +IBufferAllocationPolicy::IBufferAllocationPolicyPtr createBufferAllocationPolicy(const AzureObjectStorageSettings & settings) +{ + IBufferAllocationPolicy::Settings allocation_settings; + allocation_settings.strict_size = settings.strict_upload_part_size; + allocation_settings.min_size = settings.min_upload_part_size; + allocation_settings.max_size = settings.max_upload_part_size; + allocation_settings.multiply_factor = settings.upload_part_size_multiply_factor; + allocation_settings.multiply_parts_count_threshold = settings.upload_part_size_multiply_parts_count_threshold; + allocation_settings.max_single_size = settings.max_single_part_upload_size; + + return IBufferAllocationPolicy::create(allocation_settings); +} + WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( std::shared_ptr blob_container_client_, const String & blob_path_, @@ -27,12 +46,7 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( ThreadPoolCallbackRunner schedule_) : WriteBufferFromFileBase(buf_size_, nullptr, 0) , log(getLogger("WriteBufferFromAzureBlobStorage")) - , buffer_allocation_policy(IBufferAllocationPolicy::create({settings_->strict_upload_part_size, - settings_->min_upload_part_size, - settings_->max_upload_part_size, - settings_->upload_part_size_multiply_factor, - settings_->upload_part_size_multiply_parts_count_threshold, - settings_->max_single_part_upload_size})) + , buffer_allocation_policy(createBufferAllocationPolicy(*settings_.get())) , max_single_part_upload_size(settings_->max_single_part_upload_size) , max_unexpected_write_error_retries(settings_->max_unexpected_write_error_retries) , blob_path(blob_path_) @@ -91,6 +105,8 @@ void WriteBufferFromAzureBlobStorage::finalizeImpl() { auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); + /// If there is only one block and size is less than or equal to max_single_part_upload_size + /// then we use single part upload instead of multi part upload if (buffer_allocation_policy->getBufferNumber() == 1) { size_t data_size = size_t(position() - memory.data()); @@ -124,39 +140,30 @@ void WriteBufferFromAzureBlobStorage::allocateBuffer() { buffer_allocation_policy->nextBuffer(); auto size = buffer_allocation_policy->getBufferSize(); - - if (buffer_allocation_policy->getBufferNumber() == 1) - { - size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), size); - } - - LOG_INFO(log, "allocateBuffer size = {}", size); - memory = Memory(size); WriteBuffer::set(memory.data(), memory.size()); } void WriteBufferFromAzureBlobStorage::writePart() { - size_t data_size = size_t(position() - memory.data()); - auto data = std::move(memory); + std::shared_ptr part_data; + auto data_size = size_t(position() - memory.data()); + part_data = std::make_shared(std::move(memory), data_size); WriteBuffer::set(nullptr, 0); - if (data_size == 0) + if (part_data->data_size == 0) return; - LOG_TRACE(log, "writePart data size `{}`", data_size); - - auto upload_worker = [&] () + auto upload_worker = [&, part_data] () { auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); const std::string & block_id = block_ids.emplace_back(getRandomASCIIString(64)); - Azure::Core::IO::MemoryBodyStream memory_stream(reinterpret_cast(data.data()), data_size); - execWithRetry([&](){ block_blob_client.StageBlock(block_id, memory_stream); }, max_unexpected_write_error_retries, data_size); + Azure::Core::IO::MemoryBodyStream memory_stream(reinterpret_cast(part_data->memory.data()), part_data->data_size); + execWithRetry([&](){ block_blob_client.StageBlock(block_id, memory_stream); }, max_unexpected_write_error_retries, part_data->data_size); if (write_settings.remote_throttler) - write_settings.remote_throttler->add(data_size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); + write_settings.remote_throttler->add(part_data->data_size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); }; task_tracker->add(std::move(upload_worker)); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index c907c4cb6b6..3ee59c42079 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -74,6 +74,19 @@ struct WriteBufferFromS3::PartData } }; +IBufferAllocationPolicy::IBufferAllocationPolicyPtr createBufferAllocationPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings) +{ + IBufferAllocationPolicy::Settings allocation_settings; + allocation_settings.strict_size = settings.strict_upload_part_size; + allocation_settings.min_size = settings.min_upload_part_size; + allocation_settings.max_size = settings.max_upload_part_size; + allocation_settings.multiply_factor = settings.upload_part_size_multiply_factor; + allocation_settings.multiply_parts_count_threshold = settings.upload_part_size_multiply_parts_count_threshold; + allocation_settings.max_single_size = settings.max_single_part_upload_size; + + return IBufferAllocationPolicy::create(allocation_settings); +} + WriteBufferFromS3::WriteBufferFromS3( std::shared_ptr client_ptr_, @@ -93,12 +106,7 @@ WriteBufferFromS3::WriteBufferFromS3( , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) , object_metadata(std::move(object_metadata_)) - , buffer_allocation_policy(IBufferAllocationPolicy::create({upload_settings.strict_upload_part_size, - upload_settings.min_upload_part_size, - upload_settings.max_upload_part_size, - upload_settings.upload_part_size_multiply_factor, - upload_settings.upload_part_size_multiply_parts_count_threshold, - upload_settings.max_single_part_upload_size})) + , buffer_allocation_policy(createBufferAllocationPolicy(upload_settings)) , task_tracker( std::make_unique( std::move(schedule_), @@ -327,22 +335,10 @@ void WriteBufferFromS3::detachBuffer() detached_part_data.push_back({std::move(buf), data_size}); } -void WriteBufferFromS3::allocateFirstBuffer() -{ - const auto max_first_buffer = buffer_allocation_policy->getBufferSize(); - const auto size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), max_first_buffer); - memory = Memory(size); - WriteBuffer::set(memory.data(), memory.size()); -} - void WriteBufferFromS3::allocateBuffer() { buffer_allocation_policy->nextBuffer(); chassert(0 == hidden_size); - - if (buffer_allocation_policy->getBufferNumber() == 1) - return allocateFirstBuffer(); - memory = Memory(buffer_allocation_policy->getBufferSize()); WriteBuffer::set(memory.data(), memory.size()); } diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 201743e7f9c..3d2aed74e88 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -59,7 +59,6 @@ private: struct PartData; void hidePartialData(); - void allocateFirstBuffer(); void reallocateFirstBuffer(); void detachBuffer(); void allocateBuffer(); From 25bae3e0a5f1b7cb157eb2bc0aa9984e2e6b5274 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 20 Mar 2024 10:58:43 +0000 Subject: [PATCH 0269/1165] Verbose error medssage about analyzer_compatibility_join_using_top_level_identifier --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 30 ++++++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 0414ea93f94..c7362cc4cce 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -7409,18 +7409,40 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS if (!result_left_table_expression) result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope); - /// Here we may try to resolve identifier from projection in case it's not resolved from left table expression - /// and analyzer_compatibility_join_using_top_level_identifier is disabled. - /// For now we do not do this, because not all corner cases are clear. + /** Here we may try to resolve identifier from projection in case it's not resolved from left table expression + * and analyzer_compatibility_join_using_top_level_identifier is disabled. + * For now we do not do this, because not all corner cases are clear. + * But let's at least mention it in error message + */ /// if (!settings.analyzer_compatibility_join_using_top_level_identifier && !result_left_table_expression) /// result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope); if (!result_left_table_expression) + { + String extra_message; + const QueryNode * query_node = scope.scope_node ? scope.scope_node->as() : nullptr; + if (settings.analyzer_compatibility_join_using_top_level_identifier && query_node) + { + for (const auto & projection_node : query_node->getProjection().getNodes()) + { + if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias()) + { + extra_message = fmt::format( + " , but alias '{}' is present in SELECT list." + " You may try to SET analyzer_compatibility_join_using_top_level_identifier = 1, to allow to use it in USING clause", + projection_node->formatASTForErrorMessage()); + break; + } + } + } + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, - "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}", + "JOIN {} using identifier '{}' cannot be resolved from left table expression{}. In scope {}", join_node_typed.formatASTForErrorMessage(), identifier_full_name, + extra_message, scope.scope_node->formatASTForErrorMessage()); + } if (result_left_table_expression->getNodeType() != QueryTreeNodeType::COLUMN) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, From 5c1a54f643b9c72bf87ab03358a6244a7adf692b Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 18 Mar 2024 15:22:50 +0100 Subject: [PATCH 0270/1165] Fix `clickhouse-client -s` argument --- programs/client/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 4203e4738dd..9e90596b4f8 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -933,7 +933,7 @@ void Client::addOptions(OptionsDescription & options_description) ("config,c", po::value(), "config-file path (another shorthand)") ("connection", po::value(), "connection to use (from the client config), by default connection name is hostname") ("secure,s", "Use TLS connection") - ("no-secure,s", "Don't use TLS connection") + ("no-secure", "Don't use TLS connection") ("user,u", po::value()->default_value("default"), "user") ("password", po::value(), "password") ("ask-password", "ask-password") From 55f7595c43f7f694dca65fad2880ba92b6d5de41 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 18 Mar 2024 15:24:55 +0100 Subject: [PATCH 0271/1165] Test `-s` argument for clickhouse-client --- tests/queries/0_stateless/00505_secure.reference | 1 + tests/queries/0_stateless/00505_secure.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/00505_secure.reference b/tests/queries/0_stateless/00505_secure.reference index 9a8656bf491..73bdbdbafbe 100644 --- a/tests/queries/0_stateless/00505_secure.reference +++ b/tests/queries/0_stateless/00505_secure.reference @@ -1,3 +1,4 @@ +1 2 3 4 diff --git a/tests/queries/0_stateless/00505_secure.sh b/tests/queries/0_stateless/00505_secure.sh index b7c12911b90..dcda2c0f2e8 100755 --- a/tests/queries/0_stateless/00505_secure.sh +++ b/tests/queries/0_stateless/00505_secure.sh @@ -7,6 +7,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# Use $CLICKHOUSE_CLIENT_SECURE, but replace `--secure` by `-s` to test it +CLICKHOUSE_CLIENT_S=${CLICKHOUSE_CLIENT_SECURE/ --secure / -s } +$CLICKHOUSE_CLIENT_S -q "SELECT 1;" + $CLICKHOUSE_CLIENT_SECURE -q "SELECT 2;" #disable test From 463b90e058c1c6f0b5cd692e2fd699f5b5a1b3f8 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 18 Mar 2024 16:23:07 +0100 Subject: [PATCH 0272/1165] Add test for `--no-secure` --- tests/queries/0_stateless/00505_secure.reference | 1 + tests/queries/0_stateless/00505_secure.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/00505_secure.reference b/tests/queries/0_stateless/00505_secure.reference index 73bdbdbafbe..24319e2c136 100644 --- a/tests/queries/0_stateless/00505_secure.reference +++ b/tests/queries/0_stateless/00505_secure.reference @@ -1,3 +1,4 @@ +0 1 2 3 diff --git a/tests/queries/0_stateless/00505_secure.sh b/tests/queries/0_stateless/00505_secure.sh index dcda2c0f2e8..eaa50bce6b1 100755 --- a/tests/queries/0_stateless/00505_secure.sh +++ b/tests/queries/0_stateless/00505_secure.sh @@ -7,6 +7,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +$CLICKHOUSE_CLIENT --no-secure -q "SELECT 0;" + # Use $CLICKHOUSE_CLIENT_SECURE, but replace `--secure` by `-s` to test it CLICKHOUSE_CLIENT_S=${CLICKHOUSE_CLIENT_SECURE/ --secure / -s } $CLICKHOUSE_CLIENT_S -q "SELECT 1;" From 624b3a6dbe198eb3c125fe37593b8c22b82c78f8 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 20 Mar 2024 12:08:46 +0000 Subject: [PATCH 0273/1165] Fixing 00223_shard_distributed_aggregation_memory_efficient with analyzer. --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 17 ++++++++++++++++- src/Functions/randConstant.cpp | 16 +++++++++++----- tests/analyzer_tech_debt.txt | 1 - 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 1da5d86edf3..2a46417ee98 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -799,6 +799,8 @@ struct IdentifierResolveScope /// Node hash to mask id map std::shared_ptr> projection_mask_map; + std::map rand_constant_cache; + [[maybe_unused]] const IdentifierResolveScope * getNearestQueryScope() const { const IdentifierResolveScope * scope_to_check = this; @@ -5534,7 +5536,20 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi if (!function) { - function = FunctionFactory::instance().tryGet(function_name, scope.context); + /// This is a hack to allow a query like `select randConstant(), randConstant(), randConstant()`. + /// Function randConstant() would return the same value for the same arguments (in scope). + if (function_name == "randConstant") + { + auto hash = function_node_ptr->getTreeHash(); + auto & func = scope.rand_constant_cache[hash]; + if (!func) + func = FunctionFactory::instance().tryGet(function_name, scope.context); + + function = func; + } + else + function = FunctionFactory::instance().tryGet(function_name, scope.context); + is_executable_udf = false; } diff --git a/src/Functions/randConstant.cpp b/src/Functions/randConstant.cpp index 22ce6d88ea6..5872ab8bb34 100644 --- a/src/Functions/randConstant.cpp +++ b/src/Functions/randConstant.cpp @@ -88,6 +88,14 @@ public: return std::make_unique>(); } + RandomConstantOverloadResolver() + { + typename ColumnVector::Container vec_to(1); + + TargetSpecific::Default::RandImpl::execute(reinterpret_cast(vec_to.data()), sizeof(ToType)); + value = vec_to[0]; + } + DataTypePtr getReturnTypeImpl(const DataTypes & data_types) const override { size_t number_of_arguments = data_types.size(); @@ -105,13 +113,11 @@ public: if (!arguments.empty()) argument_types.emplace_back(arguments.back().type); - typename ColumnVector::Container vec_to(1); - - TargetSpecific::Default::RandImpl::execute(reinterpret_cast(vec_to.data()), sizeof(ToType)); - ToType value = vec_to[0]; - return std::make_unique>(value, argument_types, return_type); } + +private: + ToType value; }; struct NameRandConstant { static constexpr auto name = "randConstant"; }; diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt index 7cd73705e77..2d8497234e2 100644 --- a/tests/analyzer_tech_debt.txt +++ b/tests/analyzer_tech_debt.txt @@ -1,4 +1,3 @@ -00223_shard_distributed_aggregation_memory_efficient 00725_memory_tracking 01062_pm_all_join_with_block_continuation 01083_expressions_in_engine_arguments From 200b500ecfdb5468f9682a024e8a7903f39bb8bb Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 20 Mar 2024 13:12:58 +0100 Subject: [PATCH 0274/1165] Enable experimental analyzer by default --- src/Core/Settings.h | 2 +- src/Core/SettingsChangesHistory.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5c25368fb00..db31ace310f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -372,7 +372,7 @@ class IColumn; M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \ M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \ - M(Bool, allow_experimental_analyzer, false, "Allow experimental analyzer", 0) \ + M(Bool, allow_experimental_analyzer, true, "Allow experimental analyzer.", 0) \ M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \ M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \ M(Bool, enable_vertical_final, true, "If enable, remove duplicated rows during FINAL by marking rows as deleted and filtering them later instead of merging rows", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 0bce814170b..5c430517fa7 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -102,6 +102,7 @@ static std::map sett {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"}, {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"}, {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"}, + {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, From 15339091626617a224ec8a152696b4c4f8881c3a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 20 Mar 2024 13:22:30 +0100 Subject: [PATCH 0275/1165] Fix flaky 02122_parallel_formatting_Template --- tests/queries/0_stateless/02122_parallel_formatting_Template.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02122_parallel_formatting_Template.sh b/tests/queries/0_stateless/02122_parallel_formatting_Template.sh index c7d2329ec14..86a4c017bbe 100755 --- a/tests/queries/0_stateless/02122_parallel_formatting_Template.sh +++ b/tests/queries/0_stateless/02122_parallel_formatting_Template.sh @@ -21,7 +21,7 @@ $CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', t diff $non_parallel_file $parallel_file -echo -ne '{prefix} \n${data}\n $$ suffix $$\n${totals}\n${min}\n${max}\n${rows:Quoted}\n${rows_before_limit:Quoted}\n${rows_read:Quoted}\n${bytes_read:Quoted}\n' > $resultset_path +echo -ne '{prefix} \n${data}\n $$ suffix $$\n${totals}\n${min}\n${max}\n${rows:Quoted}\n${rows_before_limit:Quoted}\n' > $resultset_path echo "Template-2" $CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', toString(number)) as s from numbers(200000) group by number with totals order by number limit 190000 format Template $TEMPLATE_SETTINGS" --output_format_parallel_formatting=0 --extremes=1 > $non_parallel_file From f0e7b78f8be1d4b0f8c851418fd56fcccd970983 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 20 Mar 2024 13:35:30 +0100 Subject: [PATCH 0276/1165] Inverse analyzer in tests --- .github/workflows/master.yml | 4 ++-- .github/workflows/release_branches.yml | 2 +- tests/ci/functional_test_check.py | 4 ++-- tests/ci/integration_test_check.py | 2 +- tests/ci/libfuzzer_test_check.py | 2 +- tests/config/install.sh | 2 +- tests/config/users.d/analyzer.xml | 2 +- tests/integration/ci-runner.py | 4 ++-- tests/integration/helpers/0_common_enable_analyzer.xml | 2 +- tests/integration/helpers/cluster.py | 4 ++-- tests/integration/runner | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2853adff48a..816bdfd4f31 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -374,7 +374,7 @@ jobs: if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml with: - test_name: Stateless tests (release, analyzer, s3, DatabaseReplicated) + test_name: Stateless tests (release, old analyzer, s3, DatabaseReplicated) runner_type: func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestS3Debug: @@ -632,7 +632,7 @@ jobs: if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml with: - test_name: Integration tests (asan, analyzer) + test_name: Integration tests (asan, old analyzer) runner_type: stress-tester data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 9e95b3d3d8f..4d45c8d8d4b 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -436,7 +436,7 @@ jobs: if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml with: - test_name: Integration tests (asan, analyzer) + test_name: Integration tests (asan, old analyzer) runner_type: stress-tester data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 4416b7863a6..a0c977ea25f 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -39,7 +39,7 @@ def get_additional_envs( result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") result.append("RANDOMIZE_OBJECT_KEY_TYPE=1") if "analyzer" in check_name: - result.append("USE_NEW_ANALYZER=1") + result.append("USE_OLD_ANALYZER=1") if run_by_hash_total != 0: result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") @@ -94,7 +94,7 @@ def get_run_command( env_str = " ".join(envs) volume_with_broken_test = ( f"--volume={repo_path}/tests/analyzer_tech_debt.txt:/analyzer_tech_debt.txt " - if "analyzer" in check_name + if "analyzer" not in check_name else "" ) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 751abf617fa..a8701a6ec72 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -73,7 +73,7 @@ def get_env_for_runner( my_env["CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER"] = "0" if "analyzer" in check_name.lower(): - my_env["CLICKHOUSE_USE_NEW_ANALYZER"] = "1" + my_env["CLICKHOUSE_USE_OLD_ANALYZER"] = "1" return my_env diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 62dadc6fea9..4bb39010978 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -33,7 +33,7 @@ def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") result.append("RANDOMIZE_OBJECT_KEY_TYPE=1") if "analyzer" in check_name: - result.append("USE_NEW_ANALYZER=1") + result.append("USE_OLD_ANALYZER=1") if run_by_hash_total != 0: result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") diff --git a/tests/config/install.sh b/tests/config/install.sh index 467636cfa40..652d25a0a35 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -92,7 +92,7 @@ ln -sf $SRC_PATH/users.d/nonconst_timezone.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/allow_introspection_functions.yaml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/replicated_ddl_entry.xml $DEST_SERVER_PATH/users.d/ -if [[ -n "$USE_NEW_ANALYZER" ]] && [[ "$USE_NEW_ANALYZER" -eq 1 ]]; then +if [[ -n "$USE_OLD_ANALYZER" ]] && [[ "$USE_OLD_ANALYZER" -eq 1 ]]; then ln -sf $SRC_PATH/users.d/analyzer.xml $DEST_SERVER_PATH/users.d/ fi diff --git a/tests/config/users.d/analyzer.xml b/tests/config/users.d/analyzer.xml index aa374364ef0..4b9764526fa 100644 --- a/tests/config/users.d/analyzer.xml +++ b/tests/config/users.d/analyzer.xml @@ -1,7 +1,7 @@ - 1 + 0 diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index e7f691d2237..3c017db7964 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -263,7 +263,7 @@ class ClickhouseIntegrationTestsRunner: self.start_time = time.time() self.soft_deadline_time = self.start_time + (TASK_TIMEOUT - MAX_TIME_IN_SANDBOX) - self.use_analyzer = os.environ.get("CLICKHOUSE_USE_NEW_ANALYZER") is not None + self.use_analyzer = os.environ.get("CLICKHOUSE_USE_OLD_ANALYZER") is not None if "run_by_hash_total" in self.params: self.run_by_hash_total = self.params["run_by_hash_total"] @@ -916,7 +916,7 @@ class ClickhouseIntegrationTestsRunner: random.shuffle(items_to_run) broken_tests = [] - if self.use_analyzer: + if not self.use_analyzer: with open( f"{repo_path}/tests/analyzer_integration_broken_tests.txt", "r", diff --git a/tests/integration/helpers/0_common_enable_analyzer.xml b/tests/integration/helpers/0_common_enable_analyzer.xml index aa374364ef0..4b9764526fa 100644 --- a/tests/integration/helpers/0_common_enable_analyzer.xml +++ b/tests/integration/helpers/0_common_enable_analyzer.xml @@ -1,7 +1,7 @@ - 1 + 0 diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index b695b493db7..ea023bb5e97 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -4408,8 +4408,8 @@ class ClickHouseInstance: write_embedded_config("0_common_instance_users.xml", users_d_dir) if ( - os.environ.get("CLICKHOUSE_USE_NEW_ANALYZER") is not None - and self.allow_analyzer + os.environ.get("CLICKHOUSE_USE_OLD_ANALYZER") is not None + or self.allow_analyzer ): write_embedded_config("0_common_enable_analyzer.xml", users_d_dir) diff --git a/tests/integration/runner b/tests/integration/runner index f1d5198f545..b88f277cdbc 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -387,7 +387,7 @@ if __name__ == "__main__": use_analyzer = "" if args.analyzer: - use_analyzer = "-e CLICKHOUSE_USE_NEW_ANALYZER=1" + use_analyzer = "-e CLICKHOUSE_USE_OLD_ANALYZER=1" # NOTE: since pytest options is in the argument value already we need to additionally escape '"' pytest_opts = " ".join( From b74a0359e5567ae7c18ae49b55ae804298234eb1 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 20 Mar 2024 13:43:18 +0100 Subject: [PATCH 0277/1165] Update integration tests --- tests/integration/helpers/cluster.py | 10 +++++----- tests/integration/test_backward_compatibility/test.py | 4 ++-- .../test_aggregate_fixed_key.py | 6 +++--- .../test_aggregate_function_state.py | 8 ++++---- .../test_insert_profile_events.py | 4 ++-- .../test_ip_types_binary_compatibility.py | 2 +- .../test_memory_bound_aggregation.py | 6 +++--- .../test_normalized_count_comparison.py | 4 ++-- .../test_select_aggregate_alias_column.py | 4 ++-- .../test_short_strings_aggregation.py | 6 +++--- .../test_vertical_merges_from_compact_parts.py | 4 ++-- tests/integration/test_disk_over_web_server/test.py | 8 ++++---- .../test.py | 2 +- .../test_groupBitmapAnd_on_distributed/test.py | 4 ++-- .../test_groupBitmapAndState_on_distributed_table.py | 4 ++-- tests/integration/test_old_versions/test.py | 2 +- tests/integration/test_polymorphic_parts/test.py | 2 +- .../test_replicated_merge_tree_compatibility/test.py | 4 ++-- tests/integration/test_storage_s3_queue/test.py | 2 +- tests/integration/test_ttl_replicated/test.py | 6 +++--- tests/integration/test_version_update/test.py | 2 +- .../test_version_update_after_mutation/test.py | 6 +++--- 22 files changed, 50 insertions(+), 50 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index ea023bb5e97..28143cf735e 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1606,7 +1606,7 @@ class ClickHouseCluster: with_jdbc_bridge=False, with_hive=False, with_coredns=False, - allow_analyzer=True, + use_old_analyzer=False, hostname=None, env_variables=None, instance_env_variables=False, @@ -1705,7 +1705,7 @@ class ClickHouseCluster: with_coredns=with_coredns, with_cassandra=with_cassandra, with_ldap=with_ldap, - allow_analyzer=allow_analyzer, + use_old_analyzer=use_old_analyzer, server_bin_path=self.server_bin_path, odbc_bridge_bin_path=self.odbc_bridge_bin_path, library_bridge_bin_path=self.library_bridge_bin_path, @@ -3267,7 +3267,7 @@ class ClickHouseInstance: with_coredns, with_cassandra, with_ldap, - allow_analyzer, + use_old_analyzer, server_bin_path, odbc_bridge_bin_path, library_bridge_bin_path, @@ -3361,7 +3361,7 @@ class ClickHouseInstance: self.with_hive = with_hive self.with_coredns = with_coredns self.coredns_config_dir = p.abspath(p.join(base_path, "coredns_config")) - self.allow_analyzer = allow_analyzer + self.use_old_analyzer = use_old_analyzer self.main_config_name = main_config_name self.users_config_name = users_config_name @@ -4409,7 +4409,7 @@ class ClickHouseInstance: write_embedded_config("0_common_instance_users.xml", users_d_dir) if ( os.environ.get("CLICKHOUSE_USE_OLD_ANALYZER") is not None - or self.allow_analyzer + or self.use_old_analyzer ): write_embedded_config("0_common_enable_analyzer.xml", users_d_dir) diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 098fc8c1025..00973640998 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -10,13 +10,13 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index b0c0f5d17c7..263f2d0af55 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -9,10 +9,10 @@ node1 = cluster.add_instance( image="clickhouse/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) -node2 = cluster.add_instance("node2", with_zookeeper=True, allow_analyzer=False) -node3 = cluster.add_instance("node3", with_zookeeper=True, allow_analyzer=False) +node2 = cluster.add_instance("node2", with_zookeeper=True, use_old_analyzer=True) +node3 = cluster.add_instance("node3", with_zookeeper=True, use_old_analyzer=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py index 5972f57b928..ad2b20a07ad 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py @@ -10,7 +10,7 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", @@ -19,10 +19,10 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) -node3 = cluster.add_instance("node3", with_zookeeper=False, allow_analyzer=False) -node4 = cluster.add_instance("node4", with_zookeeper=False, allow_analyzer=False) +node3 = cluster.add_instance("node3", with_zookeeper=False, use_old_analyzer=True) +node4 = cluster.add_instance("node4", with_zookeeper=False, use_old_analyzer=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_backward_compatibility/test_insert_profile_events.py b/tests/integration/test_backward_compatibility/test_insert_profile_events.py index 30958127ee5..7caf15335d0 100644 --- a/tests/integration/test_backward_compatibility/test_insert_profile_events.py +++ b/tests/integration/test_backward_compatibility/test_insert_profile_events.py @@ -7,13 +7,13 @@ import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION cluster = ClickHouseCluster(__file__) -upstream_node = cluster.add_instance("upstream_node", allow_analyzer=False) +upstream_node = cluster.add_instance("upstream_node", use_old_analyzer=True) old_node = cluster.add_instance( "old_node", image="clickhouse/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py index 02c81ddbd52..f91f04da823 100644 --- a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py +++ b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py @@ -10,7 +10,7 @@ node = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py index 337a967e309..2869b91f984 100644 --- a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py @@ -10,7 +10,7 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", @@ -19,9 +19,9 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) -node3 = cluster.add_instance("node3", with_zookeeper=False, allow_analyzer=False) +node3 = cluster.add_instance("node3", with_zookeeper=False, use_old_analyzer=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py index 7f6c3fc92e1..830dec8c517 100644 --- a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py +++ b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", with_zookeeper=False, allow_analyzer=False) +node1 = cluster.add_instance("node1", with_zookeeper=False, use_old_analyzer=True) node2 = cluster.add_instance( "node2", with_zookeeper=False, @@ -11,7 +11,7 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py index be161df0640..8ddcbb71cab 100644 --- a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", with_zookeeper=False, allow_analyzer=False) +node1 = cluster.add_instance("node1", with_zookeeper=False, use_old_analyzer=True) node2 = cluster.add_instance( "node2", with_zookeeper=False, @@ -11,7 +11,7 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py index 5d0e5e24af5..715d295ac9e 100644 --- a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py @@ -10,7 +10,7 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", @@ -19,9 +19,9 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) -node3 = cluster.add_instance("node3", with_zookeeper=False, allow_analyzer=False) +node3 = cluster.add_instance("node3", with_zookeeper=False, use_old_analyzer=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index d3730357989..9740a8828b2 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -11,7 +11,7 @@ node_old = cluster.add_instance( stay_alive=True, with_installed_binary=True, with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) node_new = cluster.add_instance( "node2", @@ -21,7 +21,7 @@ node_new = cluster.add_instance( ], with_zookeeper=True, stay_alive=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_disk_over_web_server/test.py b/tests/integration/test_disk_over_web_server/test.py index dbcd7cc3c21..115161dcaad 100644 --- a/tests/integration/test_disk_over_web_server/test.py +++ b/tests/integration/test_disk_over_web_server/test.py @@ -13,7 +13,7 @@ def cluster(): "node1", main_configs=["configs/storage_conf.xml"], with_nginx=True, - allow_analyzer=False, + use_old_analyzer=True, ) cluster.add_instance( "node2", @@ -21,14 +21,14 @@ def cluster(): with_nginx=True, stay_alive=True, with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) cluster.add_instance( "node3", main_configs=["configs/storage_conf_web.xml"], with_nginx=True, with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) cluster.add_instance( @@ -39,7 +39,7 @@ def cluster(): with_installed_binary=True, image="clickhouse/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, - allow_analyzer=False, + use_old_analyzer=True, ) cluster.start() diff --git a/tests/integration/test_distributed_insert_backward_compatibility/test.py b/tests/integration/test_distributed_insert_backward_compatibility/test.py index 839e1008df1..72cc74e627a 100644 --- a/tests/integration/test_distributed_insert_backward_compatibility/test.py +++ b/tests/integration/test_distributed_insert_backward_compatibility/test.py @@ -14,7 +14,7 @@ node_dist = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_groupBitmapAnd_on_distributed/test.py b/tests/integration/test_groupBitmapAnd_on_distributed/test.py index 5119a4e0e36..118cd78105e 100644 --- a/tests/integration/test_groupBitmapAnd_on_distributed/test.py +++ b/tests/integration/test_groupBitmapAnd_on_distributed/test.py @@ -8,13 +8,13 @@ node1 = cluster.add_instance( "node1", main_configs=["configs/clusters.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", main_configs=["configs/clusters.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py b/tests/integration/test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py index 237acf6b9e0..20ba02e7a80 100644 --- a/tests/integration/test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py +++ b/tests/integration/test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py @@ -8,13 +8,13 @@ node1 = cluster.add_instance( "node1", main_configs=["configs/clusters.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", main_configs=["configs/clusters.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index 43f91b7d265..e8b4999af46 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -10,7 +10,7 @@ node_oldest = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"], - allow_analyzer=False, + use_old_analyzer=True, ) old_nodes = [node_oldest] new_node = cluster.add_instance("node_new") diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index b91a72c5534..714a89352bf 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -369,7 +369,7 @@ node7 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) node8 = cluster.add_instance( "node8", diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index 25cf3caa50c..c7e9933e1de 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -9,7 +9,7 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", @@ -18,7 +18,7 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index a7abd840834..c7893c3e643 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -108,7 +108,7 @@ def started_cluster(): tag="23.12", stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) logging.info("Starting cluster...") diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index b20b761ef47..c62e5c23438 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -23,7 +23,7 @@ node4 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, + use_old_analyzer=True, ) node5 = cluster.add_instance( @@ -36,7 +36,7 @@ node5 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, + use_old_analyzer=True, ) node6 = cluster.add_instance( "node6", @@ -48,7 +48,7 @@ node6 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_version_update/test.py b/tests/integration/test_version_update/test.py index ab3eb1860f3..704b1bd2b21 100644 --- a/tests/integration/test_version_update/test.py +++ b/tests/integration/test_version_update/test.py @@ -14,7 +14,7 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 90f8d283a6a..e9cc34a8f27 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -16,7 +16,7 @@ node1 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", @@ -28,7 +28,7 @@ node2 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, + use_old_analyzer=True, ) node3 = cluster.add_instance( "node3", @@ -40,7 +40,7 @@ node3 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, + use_old_analyzer=True, ) From 06d28eebe598e975fa14595d1ad1a5a1af3e1999 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:58:21 +0100 Subject: [PATCH 0278/1165] Fix test reference --- ...159_parallel_formatting_tsv_and_friends.reference | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference index 91e3af03db8..1131cbb670d 100644 --- a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference +++ b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference @@ -1,12 +1,12 @@ TSV, false -194d5061de4cae59489d989373f8effe - +2cc7bfde1a2855814c6ea2c8181679c8 - TSV, true -194d5061de4cae59489d989373f8effe - +2cc7bfde1a2855814c6ea2c8181679c8 - TSVWithNames, false -a6d327a3611288b3f973d00e6116f16e - +c4cb6f9c0d77cd76f2584279993b4438 - TSVWithNames, true -a6d327a3611288b3f973d00e6116f16e - +c4cb6f9c0d77cd76f2584279993b4438 - TSKV, false -c2e32a21c08aacf60bda21248ce4f73f - +fd9ccbc364c90e1f7682348fe7f11a5a - TSKV, true -c2e32a21c08aacf60bda21248ce4f73f - +fd9ccbc364c90e1f7682348fe7f11a5a - From 2c02476f4aea90906fc117067d4adc627b142cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 20 Mar 2024 14:08:00 +0100 Subject: [PATCH 0279/1165] Change riscv RLIMIT_CPU --- CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d345705ae53..f646c830daa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,11 +61,16 @@ if (ENABLE_CHECK_HEAVY_BUILDS) # set CPU time limit to 1000 seconds set (RLIMIT_CPU 1000) - # Sanitizers are too heavy. Also RISCV has some extra memory requirements - if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE OR ARCH_RISCV64) + # Sanitizers are too heavy + if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE) set (RLIMIT_DATA 10000000000) # 10G endif() + # For some files currently building RISCV64 might be too slow. TODO: Improve compilation times per file + if (ARCH_RISCV64) + set (RLIMIT_CPU 1800) + endif() + set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) endif () From 5ed895dbf529415af2896abfda72f9a096461af9 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:12:33 +0100 Subject: [PATCH 0280/1165] remove whitespace before coma --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index c7362cc4cce..6014d761794 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -7428,7 +7428,7 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias()) { extra_message = fmt::format( - " , but alias '{}' is present in SELECT list." + ", but alias '{}' is present in SELECT list." " You may try to SET analyzer_compatibility_join_using_top_level_identifier = 1, to allow to use it in USING clause", projection_node->formatASTForErrorMessage()); break; From 52b2220dc219fa7073e4cdb6ab45b37cad90df76 Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 20 Mar 2024 13:20:39 +0000 Subject: [PATCH 0281/1165] Resolved issues --- src/Common/iota.cpp | 10 +++---- src/Common/iota.h | 11 +++---- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 30 ++++++++----------- src/Storages/System/StorageSystemNumbers.cpp | 3 +- 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/Common/iota.cpp b/src/Common/iota.cpp index 86c9e04bb06..82fe86618c9 100644 --- a/src/Common/iota.cpp +++ b/src/Common/iota.cpp @@ -37,7 +37,7 @@ MULTITARGET_FUNCTION_AVX2_SSE42( ) template -void iota_with_step(T * begin, size_t count, T first_value, T step) +void iotaWithStep(T * begin, size_t count, T first_value, T step) { #if USE_MULTITARGET_CODE if (isArchSupported(TargetArch::AVX2)) @@ -56,10 +56,10 @@ template void iota(UInt64 * begin, size_t count, UInt64 first_value); template void iota(size_t * begin, size_t count, size_t first_value); #endif -template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); -template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); -template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); +template void iotaWithStep(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); +template void iotaWithStep(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); +template void iotaWithStep(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); #if defined(OS_DARWIN) -template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); +template void iotaWithStep(size_t * begin, size_t count, size_t first_value, size_t step); #endif } diff --git a/src/Common/iota.h b/src/Common/iota.h index 8fa18be9769..7ddf61736b9 100644 --- a/src/Common/iota.h +++ b/src/Common/iota.h @@ -32,12 +32,13 @@ extern template void iota(UInt64 * begin, size_t count, UInt64 first_value); extern template void iota(size_t * begin, size_t count, size_t first_value); #endif -template void iota_with_step(T * begin, size_t count, T first_value, T step); +template +void iotaWithStep(T * begin, size_t count, T first_value, T step); -extern template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); -extern template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); -extern template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); +extern template void iotaWithStep(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); +extern template void iotaWithStep(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); +extern template void iotaWithStep(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); #if defined(OS_DARWIN) -extern template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); +extern template void iotaWithStep(size_t * begin, size_t count, size_t first_value, size_t step); #endif } diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 503dd534d89..7a61d09bdd2 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -12,10 +12,7 @@ #include #include #include - -#include #include "Core/Types.h" -#include "base/Decimal_fwd.h" #include "base/types.h" namespace DB @@ -30,12 +27,12 @@ namespace { template -inline void iota_with_step_optimized(T * begin, size_t count, T first_value, T step) +inline void iotaWithStepOptimized(T * begin, size_t count, T first_value, T step) { if (step == 1) iota(begin, count, first_value); else - iota_with_step(begin, count, first_value, step); + iotaWithStep(begin, count, first_value, step); } class NumbersSource : public ISource @@ -76,7 +73,7 @@ protected: UInt64 * current_end = &vec[real_block_size]; - iota_with_step_optimized(pos, static_cast(current_end - pos), curr, step); + iotaWithStepOptimized(pos, static_cast(current_end - pos), curr, step); next += chunk_step; @@ -102,7 +99,7 @@ struct RangeWithStep using RangesWithStep = std::vector; -std::optional stepped_range_from_range(const Range & r, UInt64 step, UInt64 remainder) +std::optional steppedRangeFromRange(const Range & r, UInt64 step, UInt64 remainder) { if ((r.right.get() == 0) && (!r.right_included)) return std::nullopt; @@ -124,7 +121,7 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st return std::optional{RangeWithStep{begin, step, static_cast(right_edge_included - begin) / step + 1}}; } -[[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) +auto sizeOfRanges(const RangesWithStep & rs) { UInt128 total_size{}; for (const RangeWithStep & r : rs) @@ -138,7 +135,7 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st /// Generate numbers according to ranges. /// Numbers generated is ordered in one stream. /// Notice that we will not generate additional numbers out of ranges. -class [[maybe_unused]] NumbersRangedSource : public ISource +class NumbersRangedSource : public ISource { public: /// Represent a position in Ranges list. @@ -269,7 +266,7 @@ protected: auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = (end_value_64 - start_value_64) / this->step; - iota_with_step_optimized(pos, static_cast(size), start_value_64, step); + iotaWithStepOptimized(pos, static_cast(size), start_value_64, step); pos += size; } }; @@ -278,7 +275,7 @@ protected: { UInt64 start_value = range.left + cursor.offset_in_range * step; /// end_value will never overflow - iota_with_step_optimized(pos, static_cast(need), start_value, step); + iotaWithStepOptimized(pos, static_cast(need), start_value, step); pos += need; provided += need; cursor.offset_in_range += need; @@ -331,7 +328,7 @@ private: namespace { /// Whether we should push limit down to scan. -[[maybe_unused]] bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) +bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) { const auto & query = query_info.query->as(); /// Just ignore some minor cases, such as: @@ -344,7 +341,7 @@ namespace /// Shrink ranges to size. /// For example: ranges: [1, 5], [8, 100]; size: 7, we will get [1, 5], [8, 9] -[[maybe_unused]] void shrinkRanges(RangesWithStep & ranges, size_t size) +void shrinkRanges(RangesWithStep & ranges, size_t size) { size_t last_range_idx = 0; for (size_t i = 0; i < ranges.size(); i++) @@ -480,8 +477,8 @@ Pipe ReadFromSystemNumbersStep::makePipe() auto intersected_range = table_range->intersectWith(r); if (intersected_range.has_value()) { - auto range_with_step = stepped_range_from_range( - intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); + auto range_with_step + = steppedRangeFromRange(intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); if (range_with_step.has_value()) intersected_ranges.push_back(*range_with_step); } @@ -496,7 +493,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() auto intersected_range = overflowed_table_range->intersectWith(r); if (intersected_range) { - auto range_with_step = stepped_range_from_range( + auto range_with_step = steppedRangeFromRange( intersected_range.value(), numbers_storage.step, static_cast( @@ -518,7 +515,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() const auto & limit_length = limit_length_and_offset.first; const auto & limit_offset = limit_length_and_offset.second; - /// If intersected ranges is limited or we can pushdown limit. UInt128 total_size = sizeOfRanges(intersected_ranges); UInt128 query_limit = limit_length + limit_offset; diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index 32d16ef38f9..629b11ee7f1 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -11,7 +11,6 @@ #include #include #include -#include namespace DB { @@ -26,6 +25,8 @@ StorageSystemNumbers::StorageSystemNumbers( : IStorage(table_id), multithreaded(multithreaded_), limit(limit_), offset(offset_), column_name(column_name_), step(step_) { StorageInMemoryMetadata storage_metadata; + /// This column doesn't have a comment, because otherwise it will be added to all the tables which were created via + /// CREATE TABLE test as numbers(5) storage_metadata.setColumns(ColumnsDescription({{column_name_, std::make_shared()}})); setInMemoryMetadata(storage_metadata); } From 1524d5a4512d558888cd76b28fad685d23c63ac3 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 20 Mar 2024 14:33:14 +0100 Subject: [PATCH 0282/1165] Pass broken tests list to FastTest --- tests/ci/fast_test_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index a624ee06e85..383f5b340c7 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -47,6 +47,7 @@ def get_fasttest_cmd( f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " "-e stage=clone_submodules " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " + f"--volume={repo_path}/tests/analyzer_tech_debt.txt:/analyzer_tech_debt.txt " f"--volume={output_path}:/test_output {image}" ) From 9bf50ac814f59a1936dbe1b64a717718af1cfea8 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 20 Mar 2024 14:50:26 +0100 Subject: [PATCH 0283/1165] Fix aggregator when data is empty --- src/Interpreters/Aggregator.cpp | 7 ++++++- .../03015_aggregator_empty_data_multiple_blocks.reference | 0 .../03015_aggregator_empty_data_multiple_blocks.sql | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.reference create mode 100644 tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.sql diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 1c0d4b2d671..7f3b961a598 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1867,7 +1867,12 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Are if (data.empty()) { auto && out_cols = prepareOutputBlockColumns(params, aggregate_functions, getHeader(final), aggregates_pools, final, rows); - return {finalizeBlock(params, getHeader(final), std::move(out_cols), final, rows)}; + auto finalized_block = finalizeBlock(params, getHeader(final), std::move(out_cols), final, rows); + + if (return_single_block) + return std::move(finalized_block); + + return BlocksList{std::move(finalized_block)}; } ConvertToBlockResVariant res; bool use_compiled_functions = false; diff --git a/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.reference b/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.sql b/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.sql new file mode 100644 index 00000000000..d2c1816f7e8 --- /dev/null +++ b/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.sql @@ -0,0 +1,3 @@ +CREATE TABLE 03015_aggregator_empty_data_multiple_blocks (c0 Int32) ENGINE = Memory(); +INSERT INTO 03015_aggregator_empty_data_multiple_blocks SELECT * FROM generateRandom() LIMIT 1000; +SELECT radians(t1.c0) FROM 03015_aggregator_empty_data_multiple_blocks AS t1 RIGHT ANTI JOIN 03015_aggregator_empty_data_multiple_blocks AS right_0 ON t1.c0=right_0.c0 GROUP BY t1.c0; From cefdbfb7c3a3bd415b417b1588b82603265b56d2 Mon Sep 17 00:00:00 2001 From: Austin Kothig Date: Wed, 20 Mar 2024 07:05:49 -0700 Subject: [PATCH 0284/1165] Applying feedback. --- .../functions/date-time-functions.md | 44 +++++++++---------- src/Functions/DateTimeTransforms.h | 6 +-- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 46ec9f01152..4c419004344 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1686,17 +1686,17 @@ age('unit', startdate, enddate, [timezone]) - `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md). Possible values: - - `nanosecond` `nanoseconds` `ns` - - `microsecond` `microseconds` `us` `u` - - `millisecond` `milliseconds` `ms` - - `second` `seconds` `ss` `s` - - `minute` `minutes` `mi` `n` - - `hour` `hours` `hh` `h` - - `day` `days` `dd` `d` - - `week` `weeks` `wk` `ww` - - `month` `months` `mm` `m` - - `quarter` `quarters` `qq` `q` - - `year` `years` `yyyy` `yy` + - `nanosecond`, `nanoseconds`, `ns` + - `microsecond`, `microseconds`, `us`, `u` + - `millisecond`, `milliseconds`, `ms` + - `second`, `seconds`, `ss`, `s` + - `minute`, `minutes`, `mi`, `n` + - `hour`, `hours`, `hh`, `h` + - `day`, `days`, `dd`, `d` + - `week`, `weeks`, `wk`, `ww` + - `month`, `months`, `mm`, `m` + - `quarter`, `quarters`, `qq`, `q` + - `year`, `years`, `yyyy`, `yy` - `startdate` — The first time value to subtract (the subtrahend). [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). @@ -1764,17 +1764,17 @@ Aliases: `dateDiff`, `DATE_DIFF`, `timestampDiff`, `timestamp_diff`, `TIMESTAMP_ - `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md). Possible values: - - `nanosecond` `nanoseconds` `ns` - - `microsecond` `microseconds` `us` `u` - - `millisecond` `milliseconds` `ms` - - `second` `seconds` `ss` `s` - - `minute` `minutes` `mi` `n` - - `hour` `hours` `hh` `h` - - `day` `days` `dd` `d` - - `week` `weeks` `wk` `ww` - - `month` `months` `mm` `m` - - `quarter` `quarters` `qq` `q` - - `year` `years` `yyyy` `yy` + - `nanosecond`, `nanoseconds`, `ns` + - `microsecond`, `microseconds`, `us`, `u` + - `millisecond`, `milliseconds`, `ms` + - `second`, `seconds`, `ss`, `s` + - `minute`, `minutes`, `mi`, `n` + - `hour`, `hours`, `hh`, `h` + - `day`, `days`, `dd`, `d` + - `week`, `weeks`, `wk`, `ww` + - `month`, `months`, `mm`, `m` + - `quarter`, `quarters`, `qq`, `q` + - `year`, `years`, `yyyy`, `yy` - `startdate` — The first time value to subtract (the subtrahend). [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index a92680ac124..7410d87d16c 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -21,9 +21,9 @@ namespace DB { -static constexpr auto nanosecond_multiplier = 1000000000; -static constexpr auto microsecond_multiplier = 1000000; -static constexpr auto millisecond_multiplier = 1000; +static constexpr auto millisecond_multiplier = 1'000; +static constexpr auto microsecond_multiplier = 1'000'000; +static constexpr auto nanosecond_multiplier = 1'000'000'000; static constexpr FormatSettings::DateTimeOverflowBehavior default_date_time_overflow_behavior = FormatSettings::DateTimeOverflowBehavior::Ignore; From 981c9e0c12ef7f554f48aa7e8402eb7e15368296 Mon Sep 17 00:00:00 2001 From: Oxide Computer Company Date: Wed, 22 Nov 2023 11:47:30 +0000 Subject: [PATCH 0285/1165] Restore poco SUN files These were accidentally deleted as unused in https://github.com/ClickHouse/ClickHouse/commit/44afecf083b2cfa3d77d2e227f --- .../include/Poco/FPEnvironment_SUN.h | 75 ++++++++++ .../poco/Foundation/src/FPEnvironment_SUN.cpp | 139 ++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_SUN.h create mode 100644 base/poco/Foundation/src/FPEnvironment_SUN.cpp diff --git a/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h b/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h new file mode 100644 index 00000000000..7b31307e1ca --- /dev/null +++ b/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h @@ -0,0 +1,75 @@ +// +// FPEnvironment_SUN.h +// +// Library: Foundation +// Package: Core +// Module: FPEnvironment +// +// Definitions of class FPEnvironmentImpl for Solaris. +// +// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Foundation_FPEnvironment_SUN_INCLUDED +#define Foundation_FPEnvironment_SUN_INCLUDED + + +#include +#include "Poco/Foundation.h" + + +namespace Poco +{ + + +class FPEnvironmentImpl +{ +protected: + enum RoundingModeImpl + { + FP_ROUND_DOWNWARD_IMPL = FP_RM, + FP_ROUND_UPWARD_IMPL = FP_RP, + FP_ROUND_TONEAREST_IMPL = FP_RN, + FP_ROUND_TOWARDZERO_IMPL = FP_RZ + }; + enum FlagImpl + { + FP_DIVIDE_BY_ZERO_IMPL = FP_X_DZ, + FP_INEXACT_IMPL = FP_X_IMP, + FP_OVERFLOW_IMPL = FP_X_OFL, + FP_UNDERFLOW_IMPL = FP_X_UFL, + FP_INVALID_IMPL = FP_X_INV + }; + FPEnvironmentImpl(); + FPEnvironmentImpl(const FPEnvironmentImpl & env); + ~FPEnvironmentImpl(); + FPEnvironmentImpl & operator=(const FPEnvironmentImpl & env); + void keepCurrentImpl(); + static void clearFlagsImpl(); + static bool isFlagImpl(FlagImpl flag); + static void setRoundingModeImpl(RoundingModeImpl mode); + static RoundingModeImpl getRoundingModeImpl(); + static bool isInfiniteImpl(float value); + static bool isInfiniteImpl(double value); + static bool isInfiniteImpl(long double value); + static bool isNaNImpl(float value); + static bool isNaNImpl(double value); + static bool isNaNImpl(long double value); + static float copySignImpl(float target, float source); + static double copySignImpl(double target, double source); + static long double copySignImpl(long double target, long double source); + +private: + fp_rnd _rnd; + fp_except _exc; +}; + + +} // namespace Poco + + +#endif // Foundation_FPEnvironment_SUN_INCLUDED diff --git a/base/poco/Foundation/src/FPEnvironment_SUN.cpp b/base/poco/Foundation/src/FPEnvironment_SUN.cpp new file mode 100644 index 00000000000..36ee36431df --- /dev/null +++ b/base/poco/Foundation/src/FPEnvironment_SUN.cpp @@ -0,0 +1,139 @@ +// +// FPEnvironment_SUN.cpp +// +// Library: Foundation +// Package: Core +// Module: FPEnvironment +// +// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include +#include "Poco/FPEnvironment_SUN.h" + + +namespace Poco { + + +FPEnvironmentImpl::FPEnvironmentImpl() +{ + _rnd = fpgetround(); + _exc = fpgetmask(); +} + + +FPEnvironmentImpl::FPEnvironmentImpl(const FPEnvironmentImpl& env) +{ + _rnd = env._rnd; + _exc = env._exc; +} + + +FPEnvironmentImpl::~FPEnvironmentImpl() +{ + fpsetround(_rnd); + fpsetmask(_exc); +} + + +FPEnvironmentImpl& FPEnvironmentImpl::operator = (const FPEnvironmentImpl& env) +{ + _rnd = env._rnd; + _exc = env._exc; + return *this; +} + + +bool FPEnvironmentImpl::isInfiniteImpl(float value) +{ + int cls = fpclass(value); + return cls == FP_PINF || cls == FP_NINF; +} + + +bool FPEnvironmentImpl::isInfiniteImpl(double value) +{ + int cls = fpclass(value); + return cls == FP_PINF || cls == FP_NINF; +} + + +bool FPEnvironmentImpl::isInfiniteImpl(long double value) +{ + int cls = fpclass(value); + return cls == FP_PINF || cls == FP_NINF; +} + + +bool FPEnvironmentImpl::isNaNImpl(float value) +{ + return isnanf(value) != 0; +} + + +bool FPEnvironmentImpl::isNaNImpl(double value) +{ + return isnan(value) != 0; +} + + +bool FPEnvironmentImpl::isNaNImpl(long double value) +{ + return isnan((double) value) != 0; +} + + +float FPEnvironmentImpl::copySignImpl(float target, float source) +{ + return (float) copysign(target, source); +} + + +double FPEnvironmentImpl::copySignImpl(double target, double source) +{ + return (float) copysign(target, source); +} + + +long double FPEnvironmentImpl::copySignImpl(long double target, long double source) +{ + return (source > 0 && target > 0) || (source < 0 && target < 0) ? target : -target; +} + + +void FPEnvironmentImpl::keepCurrentImpl() +{ + fpsetround(_rnd); + fpsetmask(_exc); +} + + +void FPEnvironmentImpl::clearFlagsImpl() +{ + fpsetsticky(0); +} + + +bool FPEnvironmentImpl::isFlagImpl(FlagImpl flag) +{ + return (fpgetsticky() & flag) != 0; +} + + +void FPEnvironmentImpl::setRoundingModeImpl(RoundingModeImpl mode) +{ + fpsetround((fp_rnd) mode); +} + + +FPEnvironmentImpl::RoundingModeImpl FPEnvironmentImpl::getRoundingModeImpl() +{ + return (FPEnvironmentImpl::RoundingModeImpl) fpgetround(); +} + + +} // namespace Poco From a343406da47f62f015a5b70590fae479a3894625 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 20 Mar 2024 15:33:45 +0100 Subject: [PATCH 0286/1165] Another fix for SumIfToCountIfPass --- src/Analyzer/Passes/SumIfToCountIfPass.cpp | 10 ++-- ...3010_sum_to_to_count_if_nullable.reference | 46 +++++++++++++++++++ .../03010_sum_to_to_count_if_nullable.sql | 5 +- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp index 1c2097e7be9..1a4712aa697 100644 --- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp +++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp @@ -97,13 +97,15 @@ public: if (!if_true_condition_constant_node || !if_false_condition_constant_node) return; + if (auto constant_type = if_true_condition_constant_node->getResultType(); !isNativeInteger(constant_type)) + return; + + if (auto constant_type = if_false_condition_constant_node->getResultType(); !isNativeInteger(constant_type)) + return; + const auto & if_true_condition_constant_value_literal = if_true_condition_constant_node->getValue(); const auto & if_false_condition_constant_value_literal = if_false_condition_constant_node->getValue(); - if (!isInt64OrUInt64FieldType(if_true_condition_constant_value_literal.getType()) || - !isInt64OrUInt64FieldType(if_false_condition_constant_value_literal.getType())) - return; - auto if_true_condition_value = if_true_condition_constant_value_literal.get(); auto if_false_condition_value = if_false_condition_constant_value_literal.get(); diff --git a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference index 89e5f639c66..d20634acf93 100644 --- a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference +++ b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference @@ -1,5 +1,6 @@ (5,NULL) (5,NULL) +((6150),3) (5,NULL) QUERY id: 0 PROJECTION COLUMNS @@ -66,3 +67,48 @@ QUERY id: 0 ARGUMENTS LIST id: 19, nodes: 1 CONSTANT id: 20, constant_value: UInt64_10, constant_value_type: UInt8 +((6150),3) +QUERY id: 0 + PROJECTION COLUMNS + ((sum(if(equals(modulo(number, 2), 0), toNullable(0), 123))), toUInt8(3)) Tuple(Tuple(Nullable(UInt64)), UInt8) + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: tuple, function_type: ordinary, result_type: Tuple(Tuple(Nullable(UInt64)), UInt8) + ARGUMENTS + LIST id: 3, nodes: 2 + FUNCTION id: 4, function_name: tuple, function_type: ordinary, result_type: Tuple(Nullable(UInt64)) + ARGUMENTS + LIST id: 5, nodes: 1 + FUNCTION id: 6, function_name: sum, function_type: aggregate, nulls_action : IGNORE_NULLS, result_type: Nullable(UInt64) + ARGUMENTS + LIST id: 7, nodes: 1 + FUNCTION id: 8, function_name: if, function_type: ordinary, result_type: Nullable(UInt8) + ARGUMENTS + LIST id: 9, nodes: 3 + FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + FUNCTION id: 12, function_name: modulo, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 + COLUMN id: 14, column_name: number, result_type: UInt64, source_id: 15 + CONSTANT id: 16, constant_value: UInt64_2, constant_value_type: UInt8 + CONSTANT id: 17, constant_value: UInt64_0, constant_value_type: UInt8 + CONSTANT id: 18, constant_value: UInt64_0, constant_value_type: Nullable(UInt8) + EXPRESSION + FUNCTION id: 19, function_name: toNullable, function_type: ordinary, result_type: Nullable(UInt8) + ARGUMENTS + LIST id: 20, nodes: 1 + CONSTANT id: 21, constant_value: UInt64_0, constant_value_type: UInt8 + CONSTANT id: 22, constant_value: UInt64_123, constant_value_type: UInt8 + CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8 + EXPRESSION + FUNCTION id: 24, function_name: toUInt8, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 25, nodes: 1 + CONSTANT id: 26, constant_value: UInt64_3, constant_value_type: UInt8 + JOIN TREE + TABLE_FUNCTION id: 15, alias: __table1, table_function_name: numbers + ARGUMENTS + LIST id: 27, nodes: 1 + CONSTANT id: 28, constant_value: UInt64_100, constant_value_type: UInt8 diff --git a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql index b283a69a020..71a175faac8 100644 --- a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql +++ b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql @@ -3,9 +3,12 @@ SET optimize_rewrite_sum_if_to_count_if = 1; SET allow_experimental_analyzer = 0; SELECT (sumIf(toNullable(1), (number % 2) = 0), NULL) FROM numbers(10); SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10); +SELECT (tuple(sum(if((number % 2) = 0, toNullable(0), 123)) IGNORE NULLS), toUInt8(3)) FROM numbers(100); SET allow_experimental_analyzer = 1; SELECT (sumIf(toNullable(1), (number % 2) = 0), NULL) FROM numbers(10); EXPLAIN QUERY TREE SELECT (sumIf(toNullable(1), (number % 2) = 0), NULL) FROM numbers(10); SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10); -EXPLAIN QUERY TREE SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10); \ No newline at end of file +EXPLAIN QUERY TREE SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10); +SELECT (tuple(sum(if((number % 2) = 0, toNullable(0), 123)) IGNORE NULLS), toUInt8(3)) FROM numbers(100); +EXPLAIN QUERY TREE SELECT (tuple(sum(if((number % 2) = 0, toNullable(0), 123)) IGNORE NULLS), toUInt8(3)) FROM numbers(100); From dd1fd5ec391f91ba9656589c8af0a98e95ff3edd Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 20 Mar 2024 15:35:39 +0100 Subject: [PATCH 0287/1165] Ping CI From e6256fa380721a2d60d29658ae0f8977db1a639e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 20 Mar 2024 15:43:09 +0100 Subject: [PATCH 0288/1165] Fix build --- src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h b/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h index 7807f3d8c25..ccc88079daa 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h @@ -15,6 +15,7 @@ struct MergeTreeDataPartChecksums; struct MergeTreeIndexGranularityInfo; class ISerialization; using SerializationPtr = std::shared_ptr; +class SerializationInfoByName; /** * A class which contains all information about a data part that is required From 01c0af07aa9a93a57ca2907ccc4ba5f79c14d816 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 20 Mar 2024 14:49:07 +0000 Subject: [PATCH 0289/1165] Disable optimize_injective_functions_in_group_by when group_by_use_nulls=1 --- .../OptimizeGroupByInjectiveFunctionsPass.cpp | 13 +++++++++--- ...injective_functions_and_analyzer.reference | 21 +++++++++++++++++++ ...nulls_injective_functions_and_analyzer.sql | 5 +++++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.reference create mode 100644 tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql diff --git a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp index 11d79c7b2d9..e1a71e0e7ba 100644 --- a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp @@ -43,6 +43,13 @@ public: if (!getSettings().optimize_injective_functions_in_group_by) return; + /// Don't optimize injective functions when group_by_use_nulls=true, + /// because in this case we make initial group by keys Nullable + /// and eliminating some functions can cause issues with arguments Nullability + /// during their execution. See examples in https://github.com/ClickHouse/ClickHouse/pull/61567#issuecomment-2008181143 + if (getSettings().group_by_use_nulls) + return; + auto * query = node->as(); if (!query) return; @@ -59,15 +66,15 @@ public: for (auto & set : group_by) { auto & grouping_set = set->as()->getNodes(); - optimizeGroupingSet(grouping_set); + optimizeGroupingSet(grouping_set, getSettings().group_by_use_nulls); } } else - optimizeGroupingSet(group_by); + optimizeGroupingSet(group_by, getSettings().group_by_use_nulls); } private: - void optimizeGroupingSet(QueryTreeNodes & grouping_set) + void optimizeGroupingSet(QueryTreeNodes & grouping_set, bool) { auto context = getContext(); diff --git a/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.reference b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.reference new file mode 100644 index 00000000000..13248cf394f --- /dev/null +++ b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.reference @@ -0,0 +1,21 @@ +4 +5 +7 +19 +259 +65539 +4294967299 +\N +(((0))) +(((0))) +(((1))) +(((2))) +(((3))) +(((4))) +(((5))) +(((6))) +(((7))) +(((8))) +(((9))) +6 +6 diff --git a/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql new file mode 100644 index 00000000000..003566014a0 --- /dev/null +++ b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql @@ -0,0 +1,5 @@ +set allow_experimental_analyzer=1, group_by_use_nulls=1, optimize_injective_functions_in_group_by=1; +SELECT intExp2(intExp2(number)) + 3 FROM numbers(10) GROUP BY GROUPING SETS (('str', intExp2(intExp2(number))), ('str')) order by all; +SELECT tuple(tuple(tuple(number))) FROM numbers(10) GROUP BY GROUPING SETS (('str', tuple(tuple(number))), ('str')) order by all; +SELECT materialize(3) + 3 FROM numbers(10) GROUP BY GROUPING SETS (('str', materialize(materialize(3))), ('str')) order by all; + From 355909eb3a415b777e9402ec5963458015f53340 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 20 Mar 2024 14:54:07 +0000 Subject: [PATCH 0290/1165] Clean up --- .../OptimizeGroupByInjectiveFunctionsPass.cpp | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp index e1a71e0e7ba..61893202525 100644 --- a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp @@ -66,15 +66,15 @@ public: for (auto & set : group_by) { auto & grouping_set = set->as()->getNodes(); - optimizeGroupingSet(grouping_set, getSettings().group_by_use_nulls); + optimizeGroupingSet(grouping_set); } } else - optimizeGroupingSet(group_by, getSettings().group_by_use_nulls); + optimizeGroupingSet(group_by); } private: - void optimizeGroupingSet(QueryTreeNodes & grouping_set, bool) + void optimizeGroupingSet(QueryTreeNodes & grouping_set) { auto context = getContext(); @@ -104,20 +104,12 @@ private: if (can_be_eliminated) { - bool all_arguments_are_constants = true; for (auto const & argument : function_node->getArguments()) { // We can skip constants here because aggregation key is already not a constant. if (argument->getNodeType() != QueryTreeNodeType::CONSTANT) - { - all_arguments_are_constants = false; nodes_to_process.push(argument); - } } - - /// We cannot optimize function if it has only constant arguments (for example, materialize(const_value)). - if (all_arguments_are_constants) - new_group_by_keys.push_back(node_to_process); } else new_group_by_keys.push_back(node_to_process); From e34c4618a52b37665c04a3f3b4192add46976cf6 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 20 Mar 2024 15:56:54 +0100 Subject: [PATCH 0291/1165] Fix data race during snapshot destructor call --- src/Coordination/KeeperDispatcher.cpp | 8 +- src/Coordination/KeeperSnapshotManager.h | 3 +- src/Coordination/KeeperStateMachine.cpp | 87 ++++++++++--------- src/Coordination/tests/gtest_coordination.cpp | 2 +- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index cd642087130..84e1632e9b1 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -325,12 +325,12 @@ void KeeperDispatcher::snapshotThread() if (!snapshots_queue.pop(task)) break; - if (shutdown_called) - break; - try { - auto snapshot_file_info = task.create_snapshot(std::move(task.snapshot)); + auto snapshot_file_info = task.create_snapshot(std::move(task.snapshot), /*execute_only_cleanup=*/shutdown_called); + + if (shutdown_called) + break; if (snapshot_file_info.path.empty()) continue; diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h index ad3bcee028a..8ba0f92a564 100644 --- a/src/Coordination/KeeperSnapshotManager.h +++ b/src/Coordination/KeeperSnapshotManager.h @@ -98,8 +98,7 @@ struct SnapshotFileInfo }; using KeeperStorageSnapshotPtr = std::shared_ptr; -using CreateSnapshotCallback = std::function; - +using CreateSnapshotCallback = std::function; using SnapshotMetaAndStorage = std::pair; diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 0c398a0d549..3dbdb329b93 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -564,63 +564,65 @@ void KeeperStateMachine::create_snapshot(nuraft::snapshot & s, nuraft::async_res } /// create snapshot task for background execution (in snapshot thread) - snapshot_task.create_snapshot = [this, when_done](KeeperStorageSnapshotPtr && snapshot) + snapshot_task.create_snapshot = [this, when_done](KeeperStorageSnapshotPtr && snapshot, bool execute_only_cleanup) { nuraft::ptr exception(nullptr); bool ret = true; - try + if (!execute_only_cleanup) { - { /// Read storage data without locks and create snapshot - std::lock_guard lock(snapshots_lock); + try + { + { /// Read storage data without locks and create snapshot + std::lock_guard lock(snapshots_lock); - if (latest_snapshot_meta && snapshot->snapshot_meta->get_last_log_idx() <= latest_snapshot_meta->get_last_log_idx()) - { - LOG_INFO( - log, - "Will not create a snapshot with last log idx {} because a snapshot with bigger last log idx ({}) is already " - "created", - snapshot->snapshot_meta->get_last_log_idx(), - latest_snapshot_meta->get_last_log_idx()); - } - else - { - latest_snapshot_meta = snapshot->snapshot_meta; - /// we rely on the fact that the snapshot disk cannot be changed during runtime - if (isLocalDisk(*keeper_context->getLatestSnapshotDisk())) + if (latest_snapshot_meta && snapshot->snapshot_meta->get_last_log_idx() <= latest_snapshot_meta->get_last_log_idx()) { - auto snapshot_info = snapshot_manager.serializeSnapshotToDisk(*snapshot); - latest_snapshot_info = std::move(snapshot_info); - latest_snapshot_buf = nullptr; + LOG_INFO( + log, + "Will not create a snapshot with last log idx {} because a snapshot with bigger last log idx ({}) is already " + "created", + snapshot->snapshot_meta->get_last_log_idx(), + latest_snapshot_meta->get_last_log_idx()); } else { - auto snapshot_buf = snapshot_manager.serializeSnapshotToBuffer(*snapshot); - auto snapshot_info = snapshot_manager.serializeSnapshotBufferToDisk(*snapshot_buf, snapshot->snapshot_meta->get_last_log_idx()); - latest_snapshot_info = std::move(snapshot_info); - latest_snapshot_buf = std::move(snapshot_buf); - } + latest_snapshot_meta = snapshot->snapshot_meta; + /// we rely on the fact that the snapshot disk cannot be changed during runtime + if (isLocalDisk(*keeper_context->getLatestSnapshotDisk())) + { + auto snapshot_info = snapshot_manager.serializeSnapshotToDisk(*snapshot); + latest_snapshot_info = std::move(snapshot_info); + latest_snapshot_buf = nullptr; + } + else + { + auto snapshot_buf = snapshot_manager.serializeSnapshotToBuffer(*snapshot); + auto snapshot_info = snapshot_manager.serializeSnapshotBufferToDisk(*snapshot_buf, snapshot->snapshot_meta->get_last_log_idx()); + latest_snapshot_info = std::move(snapshot_info); + latest_snapshot_buf = std::move(snapshot_buf); + } - ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreations); - LOG_DEBUG(log, "Created persistent snapshot {} with path {}", latest_snapshot_meta->get_last_log_idx(), latest_snapshot_info.path); + ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreations); + LOG_DEBUG(log, "Created persistent snapshot {} with path {}", latest_snapshot_meta->get_last_log_idx(), latest_snapshot_info.path); + } } } - + catch (...) { - /// Destroy snapshot with lock - std::lock_guard lock(storage_and_responses_lock); - LOG_TRACE(log, "Clearing garbage after snapshot"); - /// Turn off "snapshot mode" and clear outdate part of storage state - storage->clearGarbageAfterSnapshot(); - LOG_TRACE(log, "Cleared garbage after snapshot"); - snapshot.reset(); + ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreationsFailed); + LOG_TRACE(log, "Exception happened during snapshot"); + tryLogCurrentException(log); + ret = false; } } - catch (...) { - ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreationsFailed); - LOG_TRACE(log, "Exception happened during snapshot"); - tryLogCurrentException(log); - ret = false; + /// Destroy snapshot with lock + std::lock_guard lock(storage_and_responses_lock); + LOG_TRACE(log, "Clearing garbage after snapshot"); + /// Turn off "snapshot mode" and clear outdate part of storage state + storage->clearGarbageAfterSnapshot(); + LOG_TRACE(log, "Cleared garbage after snapshot"); + snapshot.reset(); } when_done(ret, exception); @@ -628,11 +630,10 @@ void KeeperStateMachine::create_snapshot(nuraft::snapshot & s, nuraft::async_res return ret ? latest_snapshot_info : SnapshotFileInfo{}; }; - if (keeper_context->getServerState() == KeeperContext::Phase::SHUTDOWN) { LOG_INFO(log, "Creating a snapshot during shutdown because 'create_snapshot_on_exit' is enabled."); - auto snapshot_file_info = snapshot_task.create_snapshot(std::move(snapshot_task.snapshot)); + auto snapshot_file_info = snapshot_task.create_snapshot(std::move(snapshot_task.snapshot), /*execute_only_cleanup=*/false); if (!snapshot_file_info.path.empty() && snapshot_manager_s3) { diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 07dfac0670e..d314757efc9 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -1818,7 +1818,7 @@ void testLogAndStateMachine( bool pop_result = snapshots_queue.pop(snapshot_task); EXPECT_TRUE(pop_result); - snapshot_task.create_snapshot(std::move(snapshot_task.snapshot)); + snapshot_task.create_snapshot(std::move(snapshot_task.snapshot), /*execute_only_cleanup=*/false); } if (snapshot_created && changelog.size() > settings->reserved_log_items) From 87037d6d7df0bd825e40236c1f3a932865bd93de Mon Sep 17 00:00:00 2001 From: serxa Date: Wed, 20 Mar 2024 16:17:30 +0000 Subject: [PATCH 0292/1165] fix build --- src/Interpreters/DatabaseCatalog.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 61ec2d9e320..629bc6c5f4a 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -418,7 +418,9 @@ public: Names getAllRegisteredNames() const override { - return database ? database->getAllTableNames(context) : {}; + if (database) + return database->getAllTableNames(context); + return {}; } private: From a3718451b595c577658a349159d3ab71308e9b93 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 20 Mar 2024 16:36:21 +0000 Subject: [PATCH 0293/1165] stash --- src/Databases/DatabaseOnDisk.cpp | 45 +++++++++++++++++++------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index dcfc1916450..53c94cff22d 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -1,12 +1,17 @@ #include +#include +#include +#include +#include +#include #include #include #include #include +#include #include #include -#include #include #include #include @@ -15,14 +20,11 @@ #include #include #include -#include -#include -#include #include #include -#include -#include -#include +#include +#include +#include namespace fs = std::filesystem; @@ -612,7 +614,7 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat }; /// Metadata files to load: name and flag for .tmp_drop files - std::set> metadata_files; + std::vector> metadata_files; fs::directory_iterator dir_end; for (fs::directory_iterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) @@ -633,7 +635,7 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat if (endsWith(file_name, ".sql.tmp_drop")) { /// There are files that we tried to delete previously - metadata_files.emplace(file_name, false); + metadata_files.emplace_back(file_name, false); } else if (endsWith(file_name, ".sql.tmp")) { @@ -644,23 +646,30 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat else if (endsWith(file_name, ".sql")) { /// The required files have names like `table_name.sql` - metadata_files.emplace(file_name, true); + metadata_files.emplace_back(file_name, true); } else throw Exception(ErrorCodes::INCORRECT_FILE_NAME, "Incorrect file extension: {} in metadata directory {}", file_name, getMetadataPath()); } + std::sort(metadata_files.begin(), metadata_files.end()); + metadata_files.erase(std::unique(metadata_files.begin(), metadata_files.end()), metadata_files.end()); + /// Read and parse metadata in parallel ThreadPool pool(CurrentMetrics::DatabaseOnDiskThreads, CurrentMetrics::DatabaseOnDiskThreadsActive, CurrentMetrics::DatabaseOnDiskThreadsScheduled); - for (const auto & file : metadata_files) + const auto batch_size = metadata_files.size() / pool.getMaxThreads() + 1; + for (auto it = metadata_files.begin(); it < metadata_files.end(); std::advance(it, batch_size)) { - pool.scheduleOrThrowOnError([&]() - { - if (file.second) - process_metadata_file(file.first); - else - process_tmp_drop_metadata_file(file.first); - }); + std::span batch{it, std::min(std::next(it, batch_size), metadata_files.end())}; + pool.scheduleOrThrowOnError( + [batch, &process_metadata_file, &process_tmp_drop_metadata_file]() mutable + { + for (const auto & file : batch) + if (file.second) + process_metadata_file(file.first); + else + process_tmp_drop_metadata_file(file.first); + }); } pool.wait(); } From a7ff6d63c397e729919ab9e31d797b91b6ae4708 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 20 Mar 2024 16:48:24 +0000 Subject: [PATCH 0294/1165] Fix Non-ready set for system.parts. --- .../System/StorageSystemDetachedParts.cpp | 25 ++++++++++-- .../StorageSystemDroppedTablesParts.cpp | 5 ++- .../System/StorageSystemDroppedTablesParts.h | 6 +-- .../System/StorageSystemPartsBase.cpp | 39 ++++++++++++++++--- src/Storages/System/StorageSystemPartsBase.h | 6 +-- .../0_stateless/02841_not_ready_set_bug.sh | 1 + 6 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index 1eb79744022..7207e981561 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -4,10 +4,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -320,16 +322,33 @@ protected: std::shared_ptr storage; std::vector columns_mask; - const ActionsDAG::Node * predicate = nullptr; + ActionsDAGPtr filter; const size_t max_block_size; const size_t num_streams; }; void ReadFromSystemDetachedParts::applyFilters(ActionDAGNodes added_filter_nodes) { + filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); if (filter_actions_dag) - predicate = filter_actions_dag->getOutputs().at(0); + { + const auto * predicate = filter_actions_dag->getOutputs().at(0); + + Block block; + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "database")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "table")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "engine")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "active")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "uuid")); + + filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); + if (filter) + { + auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); + VirtualColumnUtils::filterBlockWithDAG(filter, empty_block, context); + } + } } void StorageSystemDetachedParts::read( @@ -358,7 +377,7 @@ void StorageSystemDetachedParts::read( void ReadFromSystemDetachedParts::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - auto state = std::make_shared(StoragesInfoStream(predicate, context)); + auto state = std::make_shared(StoragesInfoStream(nullptr, filter, context)); Pipe pipe; diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.cpp b/src/Storages/System/StorageSystemDroppedTablesParts.cpp index 20baeee1d3b..3301339df76 100644 --- a/src/Storages/System/StorageSystemDroppedTablesParts.cpp +++ b/src/Storages/System/StorageSystemDroppedTablesParts.cpp @@ -10,7 +10,7 @@ namespace DB { -StoragesDroppedInfoStream::StoragesDroppedInfoStream(const ActionsDAG::Node * predicate, ContextPtr context) +StoragesDroppedInfoStream::StoragesDroppedInfoStream(const ActionsDAGPtr & filter, ContextPtr context) : StoragesInfoStreamBase(context) { /// Will apply WHERE to subset of columns and then add more columns. @@ -73,7 +73,8 @@ StoragesDroppedInfoStream::StoragesDroppedInfoStream(const ActionsDAG::Node * pr if (block_to_filter.rows()) { /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'. - VirtualColumnUtils::filterBlockWithPredicate(predicate, block_to_filter, context); + if (filter) + VirtualColumnUtils::filterBlockWithDAG(filter, block_to_filter, context); rows = block_to_filter.rows(); } diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.h b/src/Storages/System/StorageSystemDroppedTablesParts.h index a44abea7285..dff9e41cce3 100644 --- a/src/Storages/System/StorageSystemDroppedTablesParts.h +++ b/src/Storages/System/StorageSystemDroppedTablesParts.h @@ -9,7 +9,7 @@ namespace DB class StoragesDroppedInfoStream : public StoragesInfoStreamBase { public: - StoragesDroppedInfoStream(const ActionsDAG::Node * predicate, ContextPtr context); + StoragesDroppedInfoStream(const ActionsDAGPtr & filter, ContextPtr context); protected: bool tryLockTable(StoragesInfo &) override { @@ -30,9 +30,9 @@ public: std::string getName() const override { return "SystemDroppedTablesParts"; } protected: - std::unique_ptr getStoragesInfoStream(const ActionsDAG::Node * predicate, ContextPtr context) override + std::unique_ptr getStoragesInfoStream(const ActionsDAGPtr &, const ActionsDAGPtr & filter, ContextPtr context) override { - return std::make_unique(predicate, context); + return std::make_unique(filter, context); } }; diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 6bdfdd357e8..c81c2c18b2e 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -82,7 +82,7 @@ StoragesInfo::getProjectionParts(MergeTreeData::DataPartStateVector & state, boo return data->getProjectionPartsVectorForInternalUsage({State::Active}, &state); } -StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, ContextPtr context) +StoragesInfoStream::StoragesInfoStream(const ActionsDAGPtr & filter_by_database, const ActionsDAGPtr & filter_by_other_columns, ContextPtr context) : StoragesInfoStreamBase(context) { /// Will apply WHERE to subset of columns and then add more columns. @@ -114,7 +114,8 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, Conte std::move(database_column_mut), std::make_shared(), "database")); /// Filter block_to_filter with column 'database'. - VirtualColumnUtils::filterBlockWithPredicate(predicate, block_to_filter, context); + if (filter_by_database) + VirtualColumnUtils::filterBlockWithDAG(filter_by_database, block_to_filter, context); rows = block_to_filter.rows(); /// Block contains new columns, update database_column. @@ -193,7 +194,8 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, Conte if (rows) { /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'. - VirtualColumnUtils::filterBlockWithPredicate(predicate, block_to_filter, context); + if (filter_by_other_columns) + VirtualColumnUtils::filterBlockWithDAG(filter_by_other_columns, block_to_filter, context); rows = block_to_filter.rows(); } @@ -225,7 +227,8 @@ protected: std::shared_ptr storage; std::vector columns_mask; const bool has_state_column; - const ActionsDAG::Node * predicate = nullptr; + ActionsDAGPtr filter_by_database; + ActionsDAGPtr filter_by_other_columns; }; ReadFromSystemPartsBase::ReadFromSystemPartsBase( @@ -253,7 +256,31 @@ void ReadFromSystemPartsBase::applyFilters(ActionDAGNodes added_filter_nodes) { filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); if (filter_actions_dag) - predicate = filter_actions_dag->getOutputs().at(0); + { + const auto * predicate = filter_actions_dag->getOutputs().at(0); + + Block block; + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "database")); + + filter_by_database = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); + if (filter_by_database) + { + auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); + VirtualColumnUtils::filterBlockWithDAG(filter_by_database, empty_block, context); + } + + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "table")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "engine")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "active")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "uuid")); + + filter_by_other_columns = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); + if (filter_by_other_columns) + { + auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); + VirtualColumnUtils::filterBlockWithDAG(filter_by_database, empty_block, context); + } + } } void StorageSystemPartsBase::read( @@ -287,7 +314,7 @@ void StorageSystemPartsBase::read( void ReadFromSystemPartsBase::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - auto stream = storage->getStoragesInfoStream(predicate, context); + auto stream = storage->getStoragesInfoStream(filter_by_database, filter_by_other_columns, context); auto header = getOutputStream().header; MutableColumns res_columns = header.cloneEmptyColumns(); diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h index 10d1a3a2e0e..be945162b39 100644 --- a/src/Storages/System/StorageSystemPartsBase.h +++ b/src/Storages/System/StorageSystemPartsBase.h @@ -115,7 +115,7 @@ protected: class StoragesInfoStream : public StoragesInfoStreamBase { public: - StoragesInfoStream(const ActionsDAG::Node * predicate, ContextPtr context); + StoragesInfoStream(const ActionsDAGPtr & filter_by_database, const ActionsDAGPtr & filter_by_other_columns, ContextPtr context); }; /** Implements system table 'parts' which allows to get information about data parts for tables of MergeTree family. @@ -145,9 +145,9 @@ protected: StorageSystemPartsBase(const StorageID & table_id_, ColumnsDescription && columns); - virtual std::unique_ptr getStoragesInfoStream(const ActionsDAG::Node * predicate, ContextPtr context) + virtual std::unique_ptr getStoragesInfoStream(const ActionsDAGPtr & filter_by_database, const ActionsDAGPtr & filter_by_other_columns, ContextPtr context) { - return std::make_unique(predicate, context); + return std::make_unique(filter_by_database, filter_by_other_columns, context); } virtual void diff --git a/tests/queries/0_stateless/02841_not_ready_set_bug.sh b/tests/queries/0_stateless/02841_not_ready_set_bug.sh index 3aaffe51578..9b2f3b0698e 100755 --- a/tests/queries/0_stateless/02841_not_ready_set_bug.sh +++ b/tests/queries/0_stateless/02841_not_ready_set_bug.sh @@ -10,3 +10,4 @@ $CLICKHOUSE_CLIENT -q "insert into t1 select number from numbers(10);" $CLICKHOUSE_CLIENT --max_threads=2 --max_result_rows=1 --result_overflow_mode=break -q "with tab as (select min(number) from t1 prewhere number in (select number from view(select number, row_number() OVER (partition by number % 2 ORDER BY number DESC) from numbers_mt(1e4)) where number != 2 order by number)) select number from t1 union all select * from tab;" > /dev/null $CLICKHOUSE_CLIENT -q "SELECT * FROM system.tables WHERE 1 in (SELECT number from numbers(2)) AND database = currentDatabase() format Null" +$CLICKHOUSE_CLIENT -q "SELECT xor(1, 0) FROM system.parts WHERE 1 IN (SELECT 1) FORMAT Null" From 54fae360f0a43ba644e667cefbcc28a72e0bc4f9 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 20 Mar 2024 17:50:40 +0100 Subject: [PATCH 0295/1165] Disable duplicate-inclides style check --- utils/check-style/check_cpp.sh | 9 +++++---- utils/check-style/process_style_check_result.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/check-style/check_cpp.sh b/utils/check-style/check_cpp.sh index dc1dae70e22..d647f5eefe1 100755 --- a/utils/check-style/check_cpp.sh +++ b/utils/check-style/check_cpp.sh @@ -6,10 +6,11 @@ cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_ou start_total=$(date +%s) -start=$(date +%s) -./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt -runtime=$(($(date +%s)-start)) -echo "Check duplicates. Done. $runtime seconds." +# We decided to have the regexp-based check disabled in favor of clang-tidy +# start=$(date +%s) +# ./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt +# runtime=$(($(date +%s)-start)) +# echo "Check duplicates. Done. $runtime seconds." start=$(date +%s) ./check-style -n |& tee /test_output/style_output.txt diff --git a/utils/check-style/process_style_check_result.py b/utils/check-style/process_style_check_result.py index f1ca53eba3b..e603084732d 100755 --- a/utils/check-style/process_style_check_result.py +++ b/utils/check-style/process_style_check_result.py @@ -13,7 +13,7 @@ def process_result(result_folder): description = "" test_results = [] checks = ( - "duplicate includes", + # "duplicate includes", # disabled in favor of clang-tidy "shellcheck", "style", "pylint", From 0af1bb2eaa45e08c79e2687fa8d8fe40680b78de Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 20 Mar 2024 16:58:58 +0000 Subject: [PATCH 0296/1165] Don't allow the same expression in ORDER BY with and without WITH FILL --- src/Processors/Transforms/FillingTransform.cpp | 13 ++++++++++++- .../03015_with_fill_invalid_expression.reference | 0 .../03015_with_fill_invalid_expression.sql | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03015_with_fill_invalid_expression.reference create mode 100644 tests/queries/0_stateless/03015_with_fill_invalid_expression.sql diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index c659a2578f3..54812405610 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -237,10 +237,21 @@ FillingTransform::FillingTransform( } logDebug("fill description", dumpSortDescription(fill_description)); - std::set unique_positions; + std::unordered_set ordinary_sort_positions; + for (const auto & desc : sort_description) + { + if (!desc.with_fill) + ordinary_sort_positions.insert(header_.getPositionByName(desc.column_name)); + } + + std::unordered_set unique_positions; for (auto pos : fill_column_positions) + { if (!unique_positions.insert(pos).second) throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "Multiple WITH FILL for identical expressions is not supported in ORDER BY"); + if (ordinary_sort_positions.contains(pos)) + throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "ORDER BY containing the same expression with and without WITH FILL is not supported"); + } if (use_with_fill_by_sorting_prefix) { diff --git a/tests/queries/0_stateless/03015_with_fill_invalid_expression.reference b/tests/queries/0_stateless/03015_with_fill_invalid_expression.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03015_with_fill_invalid_expression.sql b/tests/queries/0_stateless/03015_with_fill_invalid_expression.sql new file mode 100644 index 00000000000..dbb63c029d6 --- /dev/null +++ b/tests/queries/0_stateless/03015_with_fill_invalid_expression.sql @@ -0,0 +1,2 @@ +select number as x, number + 1 as y from numbers(5) where number % 3 == 1 order by y, x with fill from 1 to 4, y with fill from 2 to 5; -- {serverError INVALID_WITH_FILL_EXPRESSION} + From 952b6b85143b89cc507b6f08a46d662a7d9d87a3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 20 Mar 2024 18:00:02 +0100 Subject: [PATCH 0297/1165] Use the same features for black check --- utils/check-style/check_py.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/check-style/check_py.sh b/utils/check-style/check_py.sh index 13f4e754ed3..2e645d2f19a 100755 --- a/utils/check-style/check_py.sh +++ b/utils/check-style/check_py.sh @@ -4,9 +4,11 @@ cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_ou start_total=`date +%s` -# FIXME: 1 min to wait + head checkout -echo "Check python formatting with black" | ts +start=`date +%s` +echo "Check " | ts ./check-black -n |& tee /test_output/black_output.txt +runtime=$((`date +%s`-start)) +echo "Check python formatting with black. Done. $runtime seconds." start=`date +%s` ./check-pylint -n |& tee /test_output/pylint_output.txt From ccf6657f4dd9354fb31c7c08889eb0b51f498f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 20 Mar 2024 16:20:39 +0000 Subject: [PATCH 0298/1165] Remove unused macro --- src/Dictionaries/DictionaryStructure.h | 27 -------------------------- 1 file changed, 27 deletions(-) diff --git a/src/Dictionaries/DictionaryStructure.h b/src/Dictionaries/DictionaryStructure.h index 56d11be9837..55060b1592f 100644 --- a/src/Dictionaries/DictionaryStructure.h +++ b/src/Dictionaries/DictionaryStructure.h @@ -41,33 +41,6 @@ enum class AttributeUnderlyingType : TypeIndexUnderlying #undef map_item - -#define CALL_FOR_ALL_DICTIONARY_ATTRIBUTE_TYPES(M) \ - M(UInt8) \ - M(UInt16) \ - M(UInt32) \ - M(UInt64) \ - M(UInt128) \ - M(UInt256) \ - M(Int8) \ - M(Int16) \ - M(Int32) \ - M(Int64) \ - M(Int128) \ - M(Int256) \ - M(Decimal32) \ - M(Decimal64) \ - M(Decimal128) \ - M(Decimal256) \ - M(DateTime64) \ - M(Float32) \ - M(Float64) \ - M(UUID) \ - M(IPv4) \ - M(IPv6) \ - M(String) \ - M(Array) - /// Min and max lifetimes for a dictionary or its entry using DictionaryLifetime = ExternalLoadableLifetime; From 9e8c731e1e96ff4f8dc131f312a13ca46acb8c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 20 Mar 2024 16:22:15 +0000 Subject: [PATCH 0299/1165] Remove some inclusion of magic enum --- src/Common/ExternalLoaderStatus.cpp | 18 +++ src/Common/ExternalLoaderStatus.h | 32 ++--- src/Core/SettingsEnums.cpp | 10 ++ src/Core/SettingsEnums.h | 102 ++++++++++++++++ src/Core/SettingsFields.h | 111 ------------------ src/IO/WriteHelpers.h | 1 - src/Parsers/ASTTablesInSelectQuery.h | 2 - .../System/StorageSystemDictionaries.cpp | 2 +- 8 files changed, 142 insertions(+), 136 deletions(-) create mode 100644 src/Common/ExternalLoaderStatus.cpp diff --git a/src/Common/ExternalLoaderStatus.cpp b/src/Common/ExternalLoaderStatus.cpp new file mode 100644 index 00000000000..f3298409f43 --- /dev/null +++ b/src/Common/ExternalLoaderStatus.cpp @@ -0,0 +1,18 @@ +#include + +#include + +namespace DB +{ + +std::vector> getExternalLoaderStatusEnumAllPossibleValues() +{ + std::vector> out; + out.reserve(magic_enum::enum_count()); + + for (const auto & [value, str] : magic_enum::enum_entries()) + out.emplace_back(std::string{str}, static_cast(value)); + + return out; +} +} diff --git a/src/Common/ExternalLoaderStatus.h b/src/Common/ExternalLoaderStatus.h index 6cd27837d8f..b6e98073fea 100644 --- a/src/Common/ExternalLoaderStatus.h +++ b/src/Common/ExternalLoaderStatus.h @@ -1,30 +1,20 @@ #pragma once #include -#include #include namespace DB { - enum class ExternalLoaderStatus : int8_t - { - NOT_LOADED, /// Object hasn't been tried to load. This is an initial state. - LOADED, /// Object has been loaded successfully. - FAILED, /// Object has been failed to load. - LOADING, /// Object is being loaded right now for the first time. - FAILED_AND_RELOADING, /// Object was failed to load before and it's being reloaded right now. - LOADED_AND_RELOADING, /// Object was loaded successfully before and it's being reloaded right now. - NOT_EXIST, /// Object with this name wasn't found in the configuration. - }; +enum class ExternalLoaderStatus : int8_t +{ + NOT_LOADED, /// Object hasn't been tried to load. This is an initial state. + LOADED, /// Object has been loaded successfully. + FAILED, /// Object has been failed to load. + LOADING, /// Object is being loaded right now for the first time. + FAILED_AND_RELOADING, /// Object was failed to load before and it's being reloaded right now. + LOADED_AND_RELOADING, /// Object was loaded successfully before and it's being reloaded right now. + NOT_EXIST, /// Object with this name wasn't found in the configuration. +}; - inline std::vector> getStatusEnumAllPossibleValues() - { - std::vector> out; - out.reserve(magic_enum::enum_count()); - - for (const auto & [value, str] : magic_enum::enum_entries()) - out.emplace_back(std::string{str}, static_cast(value)); - - return out; - } +std::vector> getExternalLoaderStatusEnumAllPossibleValues(); } diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index de30d6d8eb5..c3f0715ad68 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -18,6 +18,16 @@ namespace ErrorCodes extern const int UNKNOWN_UNION; } +template +constexpr auto getEnumValues() +{ + std::array, magic_enum::enum_count()> enum_values{}; + size_t index = 0; + for (auto value : magic_enum::enum_values()) + enum_values[index++] = std::pair{magic_enum::enum_name(value), value}; + return enum_values; +} + IMPLEMENT_SETTING_ENUM(LoadBalancing, ErrorCodes::UNKNOWN_LOAD_BALANCING, {{"random", LoadBalancing::RANDOM}, {"nearest_hostname", LoadBalancing::NEAREST_HOSTNAME}, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 22fcf0389d8..c11ae75bfc7 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -12,6 +12,108 @@ namespace DB { +template +constexpr auto getEnumValues(); + +/// NOLINTNEXTLINE +#define DECLARE_SETTING_ENUM(ENUM_TYPE) \ + DECLARE_SETTING_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE) + +/// NOLINTNEXTLINE +#define DECLARE_SETTING_ENUM_WITH_RENAME(NEW_NAME, ENUM_TYPE) \ + struct SettingField##NEW_NAME##Traits \ + { \ + using EnumType = ENUM_TYPE; \ + using EnumValuePairs = std::pair[]; \ + static const String & toString(EnumType value); \ + static EnumType fromString(std::string_view str); \ + }; \ + \ + using SettingField##NEW_NAME = SettingFieldEnum; + +/// NOLINTNEXTLINE +#define IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ + IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, EnumValuePairs, __VA_ARGS__) + +/// NOLINTNEXTLINE +#define IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \ + IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, , getEnumValues()) + +/// NOLINTNEXTLINE +#define IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, PAIRS_TYPE, ...) \ + const String & SettingField##NEW_NAME##Traits::toString(typename SettingField##NEW_NAME::EnumType value) \ + { \ + static const std::unordered_map map = [] { \ + std::unordered_map res; \ + for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \ + res.emplace(val, name); \ + return res; \ + }(); \ + auto it = map.find(value); \ + if (it != map.end()) \ + return it->second; \ + throw Exception(ERROR_CODE_FOR_UNEXPECTED_NAME, \ + "Unexpected value of " #NEW_NAME ":{}", std::to_string(std::underlying_type_t(value))); \ + } \ + \ + typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \ + { \ + static const std::unordered_map map = [] { \ + std::unordered_map res; \ + for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \ + res.emplace(name, val); \ + return res; \ + }(); \ + auto it = map.find(str); \ + if (it != map.end()) \ + return it->second; \ + String msg; \ + bool need_comma = false; \ + for (auto & name : map | boost::adaptors::map_keys) \ + { \ + if (std::exchange(need_comma, true)) \ + msg += ", "; \ + msg += "'" + String{name} + "'"; \ + } \ + throw Exception(ERROR_CODE_FOR_UNEXPECTED_NAME, "Unexpected value of " #NEW_NAME ": '{}'. Must be one of [{}]", String{str}, msg); \ + } + +/// NOLINTNEXTLINE +#define DECLARE_SETTING_MULTI_ENUM(ENUM_TYPE) \ + DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE) + +/// NOLINTNEXTLINE +#define DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, NEW_NAME) \ + struct SettingField##NEW_NAME##Traits \ + { \ + using EnumType = ENUM_TYPE; \ + using EnumValuePairs = std::pair[]; \ + static size_t getEnumSize(); \ + static const String & toString(EnumType value); \ + static EnumType fromString(std::string_view str); \ + }; \ + \ + using SettingField##NEW_NAME = SettingFieldMultiEnum; \ + using NEW_NAME##List = typename SettingField##NEW_NAME::ValueType; + +/// NOLINTNEXTLINE +#define IMPLEMENT_SETTING_MULTI_ENUM(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ + IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__) + +/// NOLINTNEXTLINE +#define IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ + IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)\ + size_t SettingField##NEW_NAME##Traits::getEnumSize() {\ + return std::initializer_list> __VA_ARGS__ .size();\ + } + +/// NOLINTNEXTLINE +#define IMPLEMENT_SETTING_MULTI_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \ + IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME)\ + size_t SettingField##NEW_NAME##Traits::getEnumSize() {\ + return getEnumValues().size();\ + } + enum class LoadBalancing { /// among replicas with a minimum number of errors selected randomly diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index 452f3f149ab..cef30bb1916 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -7,9 +7,7 @@ #include #include #include -#include #include -#include namespace DB @@ -380,79 +378,6 @@ void SettingFieldEnum::readBinary(ReadBuffer & in) *this = Traits::fromString(SettingFieldEnumHelpers::readBinary(in)); } -template -constexpr auto getEnumValues() -{ - std::array, magic_enum::enum_count()> enum_values{}; - size_t index = 0; - for (auto value : magic_enum::enum_values()) - enum_values[index++] = std::pair{magic_enum::enum_name(value), value}; - return enum_values; -} - -/// NOLINTNEXTLINE -#define DECLARE_SETTING_ENUM(ENUM_TYPE) \ - DECLARE_SETTING_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE) - -/// NOLINTNEXTLINE -#define DECLARE_SETTING_ENUM_WITH_RENAME(NEW_NAME, ENUM_TYPE) \ - struct SettingField##NEW_NAME##Traits \ - { \ - using EnumType = ENUM_TYPE; \ - using EnumValuePairs = std::pair[]; \ - static const String & toString(EnumType value); \ - static EnumType fromString(std::string_view str); \ - }; \ - \ - using SettingField##NEW_NAME = SettingFieldEnum; - -/// NOLINTNEXTLINE -#define IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ - IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, EnumValuePairs, __VA_ARGS__) - -/// NOLINTNEXTLINE -#define IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \ - IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, , getEnumValues()) - -/// NOLINTNEXTLINE -#define IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, PAIRS_TYPE, ...) \ - const String & SettingField##NEW_NAME##Traits::toString(typename SettingField##NEW_NAME::EnumType value) \ - { \ - static const std::unordered_map map = [] { \ - std::unordered_map res; \ - for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \ - res.emplace(val, name); \ - return res; \ - }(); \ - auto it = map.find(value); \ - if (it != map.end()) \ - return it->second; \ - throw Exception(ERROR_CODE_FOR_UNEXPECTED_NAME, \ - "Unexpected value of " #NEW_NAME ":{}", std::to_string(std::underlying_type_t(value))); \ - } \ - \ - typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \ - { \ - static const std::unordered_map map = [] { \ - std::unordered_map res; \ - for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \ - res.emplace(name, val); \ - return res; \ - }(); \ - auto it = map.find(str); \ - if (it != map.end()) \ - return it->second; \ - String msg; \ - bool need_comma = false; \ - for (auto & name : map | boost::adaptors::map_keys) \ - { \ - if (std::exchange(need_comma, true)) \ - msg += ", "; \ - msg += "'" + String{name} + "'"; \ - } \ - throw Exception(ERROR_CODE_FOR_UNEXPECTED_NAME, "Unexpected value of " #NEW_NAME ": '{}'. Must be one of [{}]", String{str}, msg); \ - } - // Mostly like SettingFieldEnum, but can have multiple enum values (or none) set at once. template struct SettingFieldMultiEnum @@ -543,42 +468,6 @@ void SettingFieldMultiEnum::readBinary(ReadBuffer & in) parseFromString(SettingFieldEnumHelpers::readBinary(in)); } -/// NOLINTNEXTLINE -#define DECLARE_SETTING_MULTI_ENUM(ENUM_TYPE) \ - DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE) - -/// NOLINTNEXTLINE -#define DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, NEW_NAME) \ - struct SettingField##NEW_NAME##Traits \ - { \ - using EnumType = ENUM_TYPE; \ - using EnumValuePairs = std::pair[]; \ - static size_t getEnumSize(); \ - static const String & toString(EnumType value); \ - static EnumType fromString(std::string_view str); \ - }; \ - \ - using SettingField##NEW_NAME = SettingFieldMultiEnum; \ - using NEW_NAME##List = typename SettingField##NEW_NAME::ValueType; - -/// NOLINTNEXTLINE -#define IMPLEMENT_SETTING_MULTI_ENUM(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ - IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__) - -/// NOLINTNEXTLINE -#define IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ - IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)\ - size_t SettingField##NEW_NAME##Traits::getEnumSize() {\ - return std::initializer_list> __VA_ARGS__ .size();\ - } - -/// NOLINTNEXTLINE -#define IMPLEMENT_SETTING_MULTI_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \ - IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME)\ - size_t SettingField##NEW_NAME##Traits::getEnumSize() {\ - return getEnumValues().size();\ - } - /// Setting field for specifying user-defined timezone. It is basically a string, but it needs validation. struct SettingFieldTimezone { diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 8b743e6351b..b42b4e6e978 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -19,7 +19,6 @@ #include #include #include -#include #include #include diff --git a/src/Parsers/ASTTablesInSelectQuery.h b/src/Parsers/ASTTablesInSelectQuery.h index 67370eaee14..39ef0b21f6d 100644 --- a/src/Parsers/ASTTablesInSelectQuery.h +++ b/src/Parsers/ASTTablesInSelectQuery.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include diff --git a/src/Storages/System/StorageSystemDictionaries.cpp b/src/Storages/System/StorageSystemDictionaries.cpp index 8b528b4a298..a62a4a3f425 100644 --- a/src/Storages/System/StorageSystemDictionaries.cpp +++ b/src/Storages/System/StorageSystemDictionaries.cpp @@ -66,7 +66,7 @@ ColumnsDescription StorageSystemDictionaries::getColumnsDescription() {"database", std::make_shared(), "Name of the database containing the dictionary created by DDL query. Empty string for other dictionaries."}, {"name", std::make_shared(), "Dictionary name."}, {"uuid", std::make_shared(), "Dictionary UUID."}, - {"status", std::make_shared(getStatusEnumAllPossibleValues()), + {"status", std::make_shared(getExternalLoaderStatusEnumAllPossibleValues()), "Dictionary status. Possible values: " "NOT_LOADED — Dictionary was not loaded because it was not used, " "LOADED — Dictionary loaded successfully, " From 0b54b34b5b8339b163c17420e552aa1dbca2692d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 20 Mar 2024 16:27:32 +0000 Subject: [PATCH 0300/1165] Avoid instantiation of `CacheBase`'s ctor in header file --- src/IO/UncompressedCache.cpp | 11 +++++++++++ src/IO/UncompressedCache.h | 4 ++-- src/Storages/MarkCache.cpp | 11 +++++++++++ src/Storages/MarkCache.h | 5 ++--- 4 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 src/IO/UncompressedCache.cpp create mode 100644 src/Storages/MarkCache.cpp diff --git a/src/IO/UncompressedCache.cpp b/src/IO/UncompressedCache.cpp new file mode 100644 index 00000000000..7309ef5d2f4 --- /dev/null +++ b/src/IO/UncompressedCache.cpp @@ -0,0 +1,11 @@ +#include + +namespace DB +{ +template class CacheBase; + +UncompressedCache::UncompressedCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) + : Base(cache_policy, max_size_in_bytes, 0, size_ratio) +{ +} +} diff --git a/src/IO/UncompressedCache.h b/src/IO/UncompressedCache.h index 702804cdda3..aa515eec357 100644 --- a/src/IO/UncompressedCache.h +++ b/src/IO/UncompressedCache.h @@ -33,6 +33,7 @@ struct UncompressedSizeWeightFunction } }; +extern template class CacheBase; /** Cache of decompressed blocks for implementation of CachedCompressedReadBuffer. thread-safe. */ @@ -42,8 +43,7 @@ private: using Base = CacheBase; public: - UncompressedCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) - : Base(cache_policy, max_size_in_bytes, 0, size_ratio) {} + UncompressedCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); /// Calculate key from path to file and offset. static UInt128 hash(const String & path_to_file, size_t offset) diff --git a/src/Storages/MarkCache.cpp b/src/Storages/MarkCache.cpp new file mode 100644 index 00000000000..85f0aeac692 --- /dev/null +++ b/src/Storages/MarkCache.cpp @@ -0,0 +1,11 @@ +#include + +namespace DB +{ +template class CacheBase; + +MarkCache::MarkCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) + : Base(cache_policy, max_size_in_bytes, 0, size_ratio) +{ +} +} diff --git a/src/Storages/MarkCache.h b/src/Storages/MarkCache.h index bcb6ae96c68..311fccdf810 100644 --- a/src/Storages/MarkCache.h +++ b/src/Storages/MarkCache.h @@ -31,7 +31,7 @@ struct MarksWeightFunction } }; - +extern template class CacheBase; /** Cache of 'marks' for StorageMergeTree. * Marks is an index structure that addresses ranges in column file, corresponding to ranges of primary key. */ @@ -41,8 +41,7 @@ private: using Base = CacheBase; public: - MarkCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) - : Base(cache_policy, max_size_in_bytes, 0, size_ratio) {} + MarkCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); /// Calculate key from path to file and offset. static UInt128 hash(const String & path_to_file) From ed9db6d392ed8b68c6d39ba8ffab81f875bc6949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 20 Mar 2024 18:08:58 +0100 Subject: [PATCH 0301/1165] Add fuzz queries with CUBE and ROLLUP --- ...03014_analyzer_groupby_fuzz_60317.reference | 10 ++++++++++ .../03014_analyzer_groupby_fuzz_60317.sql | 18 ++++++++++++++++++ ...03015_analyzer_groupby_fuzz_60772.reference | 8 ++++++++ .../03015_analyzer_groupby_fuzz_60772.sql | 13 +++++++++++++ ...03017_analyzer_groupby_fuzz_61600.reference | 3 +++ .../03017_analyzer_groupby_fuzz_61600.sql | 11 +++++++++++ 6 files changed, 63 insertions(+) diff --git a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference index 4972904f87d..5e56482a470 100644 --- a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference +++ b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference @@ -1 +1,11 @@ 30 30 1970-01-01 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 + +30 1970-01-01 30 30 1970-01-01 -1980.1 diff --git a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql index 2f54058526e..094614cb78d 100644 --- a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql +++ b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql @@ -7,3 +7,21 @@ FROM system.one GROUP BY _CAST(30, 'Nullable(UInt8)') SETTINGS allow_experimental_analyzer = 1; + +-- WITH CUBE (note that result is different with the analyzer (analyzer is correct including all combinations) +SELECT + toNullable(toNullable(materialize(_CAST(30, 'LowCardinality(UInt8)')))) AS `toNullable(toNullable(materialize(toLowCardinality(30))))`, + _CAST(0, 'Date') AS `makeDate(-1980.1, -1980.1, 10)`, + _CAST(30, 'LowCardinality(UInt8)') AS `toLowCardinality(30)`, + 30 AS `30`, + makeDate(materialize(_CAST(30, 'LowCardinality(UInt8)')), 10, _CAST(30, 'Nullable(UInt8)')) AS `makeDate(materialize(toLowCardinality(30)), 10, toNullable(toNullable(30)))`, + -1980.1 AS `-1980.1` +FROM system.one AS __table1 +GROUP BY + _CAST(30, 'Nullable(UInt8)'), + -1980.1, + materialize(30), + _CAST(30, 'Nullable(UInt8)') +WITH CUBE +WITH TOTALS +SETTINGS allow_experimental_analyzer = 1; diff --git a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference index 611407ecd90..5cdf256a24a 100644 --- a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference +++ b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference @@ -1 +1,9 @@ %W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 diff --git a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql index c8b4eef50ff..d3bd9ef0ce3 100644 --- a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql +++ b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql @@ -8,3 +8,16 @@ GROUP BY 'gr', '2018-01-02 22:33:44' SETTINGS allow_experimental_analyzer = 1; + +-- WITH CUBE (note that result is different with the analyzer (analyzer is correct including all combinations) +SELECT + toFixedString(toFixedString(toFixedString(toFixedString(toFixedString(toFixedString('%W', 2), 2), 2), toLowCardinality(toLowCardinality(toNullable(2)))), 2), 2), + toFixedString(toFixedString('2018-01-02 22:33:44', 19), 19), + hasSubsequence(toNullable(materialize(toLowCardinality('garbage'))), 'gr') +GROUP BY + '2018-01-02 22:33:44', + toFixedString(toFixedString('2018-01-02 22:33:44', 19), 19), + 'gr', + '2018-01-02 22:33:44' +WITH CUBE +SETTINGS allow_experimental_analyzer = 1; diff --git a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference index 68acb650f8e..eaa1097a734 100644 --- a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference +++ b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference @@ -1 +1,4 @@ 38 \N +38 \N +38 \N +38 \N diff --git a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql index 44b27750c16..53a5cfe9b1a 100644 --- a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql +++ b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql @@ -12,3 +12,14 @@ GROUP BY toNullable(3), concat(concat(NULLIF(1, 1), toNullable(toNullable(3)))) SETTINGS allow_experimental_analyzer = 1; + +-- WITH ROLLUP (note that result is different with the analyzer (analyzer is correct including all combinations) +SELECT + 38, + concat(position(concat(concat(position(concat(toUInt256(3)), 'ca', 2), 3), NULLIF(1, materialize(toLowCardinality(1)))), toLowCardinality(toNullable('ca'))), concat(NULLIF(1, 1), concat(3), toNullable(3))) +FROM set_index_not__fuzz_0 +GROUP BY + toNullable(3), + concat(concat(NULLIF(1, 1), toNullable(toNullable(3)))) +WITH ROLLUP +SETTINGS allow_experimental_analyzer = 1; From c799bfd016f9f537df2626a178eaba2f4de8a5b2 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 19 Jan 2024 16:57:44 +0000 Subject: [PATCH 0302/1165] Analyzer: fixes for join using columns --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 124 +++++++++++++++++- src/Interpreters/ActionsDAG.cpp | 2 +- src/Planner/CollectTableExpressionData.cpp | 28 +++- src/Planner/PlannerJoinTree.cpp | 44 ++++--- .../0_stateless/00722_inner_join.reference | 6 +- .../02514_analyzer_drop_join_on.reference | 36 ++--- .../02722_matcher_join_use_nulls.reference | 16 +-- .../02835_join_step_explain.reference | 36 ++--- ...5_analyzer_using_functional_args.reference | 79 +++++++++++ .../02955_analyzer_using_functional_args.sql | 53 +++++++- ..._join_use_nulls_column_not_found.reference | 2 + ...alyzer_join_use_nulls_column_not_found.sql | 31 +++++ 12 files changed, 373 insertions(+), 84 deletions(-) create mode 100644 tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.reference create mode 100644 tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.sql diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 5a557875fec..5058a194239 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1350,6 +1350,7 @@ private: const NamesAndTypes & matched_columns, const IdentifierResolveScope & scope); + void updateMatchedColumnsFromJoinUsing(QueryTreeNodesWithNames & result_matched_column_nodes_with_names, const QueryTreeNodePtr & source_table_expression, IdentifierResolveScope & scope); QueryTreeNodesWithNames resolveQualifiedMatcher(QueryTreeNodePtr & matcher_node, IdentifierResolveScope & scope); QueryTreeNodesWithNames resolveUnqualifiedMatcher(QueryTreeNodePtr & matcher_node, IdentifierResolveScope & scope); @@ -3272,6 +3273,9 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo auto & result_column = result_column_node->as(); result_column.setColumnType(using_column_node.getColumnType()); + /// Reset the expression for the column in the projection (or any other section outside of JOIN), it is calculated beforehand for JOIN USING. + result_column.setExpression(nullptr); + resolved_identifier = std::move(result_column_node); } else if (left_resolved_identifier->isEqual(*right_resolved_identifier, IQueryTreeNode::CompareOptions{.compare_aliases = false})) @@ -3344,11 +3348,15 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo { auto & left_resolved_column = left_resolved_identifier->as(); auto using_column_node_it = join_using_column_name_to_column_node.find(left_resolved_column.getColumnName()); - if (using_column_node_it != join_using_column_name_to_column_node.end() && - !using_column_node_it->second->getColumnType()->equals(*left_resolved_column.getColumnType())) + if (using_column_node_it != join_using_column_name_to_column_node.end()) { auto left_resolved_column_clone = std::static_pointer_cast(left_resolved_column.clone()); - left_resolved_column_clone->setColumnType(using_column_node_it->second->getColumnType()); + if (!using_column_node_it->second->getColumnType()->equals(*left_resolved_column.getColumnType())) + left_resolved_column_clone->setColumnType(using_column_node_it->second->getColumnType()); + + /// Reset the expression for the column in the projection (or any other section outside of JOIN), it is calculated beforehand for JOIN USING. + left_resolved_column_clone->setExpression(nullptr); + resolved_identifier = std::move(left_resolved_column_clone); } } @@ -3366,11 +3374,15 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo { auto & right_resolved_column = right_resolved_identifier->as(); auto using_column_node_it = join_using_column_name_to_column_node.find(right_resolved_column.getColumnName()); - if (using_column_node_it != join_using_column_name_to_column_node.end() && - !using_column_node_it->second->getColumnType()->equals(*right_resolved_column.getColumnType())) + if (using_column_node_it != join_using_column_name_to_column_node.end()) { auto right_resolved_column_clone = std::static_pointer_cast(right_resolved_column.clone()); - right_resolved_column_clone->setColumnType(using_column_node_it->second->getColumnType()); + if (!using_column_node_it->second->getColumnType()->equals(*right_resolved_column.getColumnType())) + right_resolved_column_clone->setColumnType(using_column_node_it->second->getColumnType()); + + /// Reset the expression for the column in the projection (or any other section outside of JOIN), it is calculated beforehand for JOIN USING. + right_resolved_column_clone->setExpression(nullptr); + resolved_identifier = std::move(right_resolved_column_clone); } } @@ -3381,7 +3393,10 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo if (join_use_nulls) { + auto it = node_to_projection_name.find(resolved_identifier); resolved_identifier = resolved_identifier->clone(); + if (it != node_to_projection_name.end()) + node_to_projection_name.emplace(resolved_identifier, it->second); convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side); } @@ -3955,6 +3970,97 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::getMatchedColumnNodesWithN return matched_column_nodes_with_names; } + +bool hasTableExpressionInJoinTree(const QueryTreeNodePtr & join_tree_node, const QueryTreeNodePtr & table_expression) +{ + QueryTreeNodes nodes_to_process; + nodes_to_process.push_back(join_tree_node); + + while (!nodes_to_process.empty()) + { + auto node_to_process = std::move(nodes_to_process.back()); + nodes_to_process.pop_back(); + if (node_to_process == table_expression) + return true; + + if (node_to_process->getNodeType() == QueryTreeNodeType::JOIN) + { + const auto & join_node = node_to_process->as(); + nodes_to_process.push_back(join_node.getLeftTableExpression()); + nodes_to_process.push_back(join_node.getRightTableExpression()); + } + } + return false; +} + +/// Columns that resolved from matcher can also match columns from JOIN USING. +/// In that case we update type to type of column in USING section. +/// TODO: It's not completely correct for qualified matchers, so t1.* should be resolved to left table column type. +/// But in planner we do not distinguish such cases. +void QueryAnalyzer::updateMatchedColumnsFromJoinUsing( + QueryTreeNodesWithNames & result_matched_column_nodes_with_names, + const QueryTreeNodePtr & source_table_expression, + IdentifierResolveScope & scope) +{ + auto * nearest_query_scope = scope.getNearestQueryScope(); + auto * nearest_query_scope_query_node = nearest_query_scope ? nearest_query_scope->scope_node->as() : nullptr; + + /// If there are no parent query scope or query scope does not have join tree + if (!nearest_query_scope_query_node || !nearest_query_scope_query_node->getJoinTree()) + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "There are no table sources. In scope {}", + scope.scope_node->formatASTForErrorMessage()); + } + + const auto & join_tree = nearest_query_scope_query_node->getJoinTree(); + + const auto * join_node = join_tree->as(); + if (join_node && join_node->isUsingJoinExpression()) + { + const auto & join_using_list = join_node->getJoinExpression()->as(); + const auto & join_using_nodes = join_using_list.getNodes(); + + for (auto & [matched_column_node, _] : result_matched_column_nodes_with_names) + { + auto & matched_column_node_typed = matched_column_node->as(); + const auto & matched_column_name = matched_column_node_typed.getColumnName(); + + for (const auto & join_using_node : join_using_nodes) + { + auto & join_using_column_node = join_using_node->as(); + const auto & join_using_column_name = join_using_column_node.getColumnName(); + + if (matched_column_name != join_using_column_name) + continue; + + const auto & join_using_column_nodes_list = join_using_column_node.getExpressionOrThrow()->as(); + const auto & join_using_column_nodes = join_using_column_nodes_list.getNodes(); + + auto it = node_to_projection_name.find(matched_column_node); + + if (hasTableExpressionInJoinTree(join_node->getLeftTableExpression(), source_table_expression)) + matched_column_node = join_using_column_nodes.at(0); + else if (hasTableExpressionInJoinTree(join_node->getRightTableExpression(), source_table_expression)) + matched_column_node = join_using_column_nodes.at(1); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot find column {} in JOIN USING section {}", + matched_column_node->dumpTree(), join_node->dumpTree()); + + matched_column_node = matched_column_node->clone(); + if (it != node_to_projection_name.end()) + node_to_projection_name.emplace(matched_column_node, it->second); + + matched_column_node->as().setColumnType(join_using_column_node.getResultType()); + + /// Reset the expression for the column in the projection (or any other section outside of JOIN), it is calculated beforehand for JOIN USING. + matched_column_node->as().setExpression(nullptr); + } + } + } +} + /** Resolve qualified tree matcher. * * First try to match qualified identifier to expression. If qualified identifier matched expression node then @@ -4049,7 +4155,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveQualifiedMatcher(Qu if (table_expression_query_node || table_expression_union_node) { matched_columns = table_expression_query_node ? table_expression_query_node->getProjectionColumns() - : table_expression_union_node->computeProjectionColumns(); + : table_expression_union_node->computeProjectionColumns(); } else if (table_expression_table_node || table_expression_table_function_node) { @@ -4072,6 +4178,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveQualifiedMatcher(Qu matched_columns, scope); + updateMatchedColumnsFromJoinUsing(result_matched_column_nodes_with_names, table_expression_node, scope); return result_matched_column_nodes_with_names; } @@ -4186,6 +4293,9 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( matched_column_node = matched_column_node->clone(); matched_column_node->as().setColumnType(join_using_column_node.getResultType()); + /// Reset the expression for the column in the projection (or any other section outside of JOIN), it is calculated beforehand for JOIN USING. + matched_column_node->as().setExpression(nullptr); + table_expression_column_names_to_skip.insert(join_using_column_name); matched_expression_nodes_with_column_names.emplace_back(std::move(matched_column_node), join_using_column_name); } diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 1124ba94bc1..4a5eb514bb0 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -679,7 +679,7 @@ Block ActionsDAG::updateHeader(Block header) const catch (Exception & e) { if (e.code() == ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK) - e.addMessage(" in block {}", header.dumpStructure()); + e.addMessage("in block {}", header.dumpStructure()); throw; } diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp index 78a7c7074c3..b87446a3ea2 100644 --- a/src/Planner/CollectTableExpressionData.cpp +++ b/src/Planner/CollectTableExpressionData.cpp @@ -91,6 +91,11 @@ public: return; } + /// JoinNode doesn't have its dedicated identifiers, + /// we apply projection actions from outer scope to columns from JOIN + if (column_source_node_type == QueryTreeNodeType::JOIN) + return; + if (column_source_node_type != QueryTreeNodeType::TABLE && column_source_node_type != QueryTreeNodeType::TABLE_FUNCTION && column_source_node_type != QueryTreeNodeType::QUERY && @@ -123,7 +128,7 @@ private: void visitUsingColumn(QueryTreeNodePtr & node) { - auto & column_node = node->as(); + auto & column_node = node->as(); if (column_node.hasExpression()) { auto & table_expression_data = planner_context.getOrCreateTableExpressionData(column_node.getColumnSource()); @@ -134,10 +139,10 @@ private: auto column_identifier = planner_context.getGlobalPlannerContext()->createColumnIdentifier(node); table_expression_data.addAliasColumnName(column_node.getColumnName(), column_identifier); - visitImpl(column_node.getExpressionOrThrow()); + visit(column_node.getExpressionOrThrow()); } else - visitImpl(node); + visit(node); } PlannerContext & planner_context; @@ -275,13 +280,28 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr } CollectSourceColumnsVisitor collect_source_columns_visitor(*planner_context); + + /** Visit the join nodes first to ensure that all columns in subtrees are collected before the projection node is visited. + * This is crucial for column nodes that originate from the JOIN USING clause. + * For example: + * SELECT a FROM t1 JOIN t2 USING a ORDER BY a; + * In this case, if `a` is an ALIAS column, the expression for it will be calculated from the USING clause. + * Therefore, the table expression data for t1 and t2 should be prepared before visiting the `a` column node in projection or ORDER BY. + */ + auto table_expression_stack = buildTableExpressionsStack(query_node_typed.getJoinTree()); + for (auto & table_expression_node : table_expression_stack) + { + if (table_expression_node->getNodeType() == QueryTreeNodeType::JOIN) + collect_source_columns_visitor.visit(table_expression_node); + } + for (auto & node : query_node_typed.getChildren()) { if (!node || node == query_node_typed.getPrewhere()) continue; auto node_type = node->getNodeType(); - if (node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION) + if (node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION || node_type == QueryTreeNodeType::JOIN) continue; collect_source_columns_visitor.visit(node); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index ab25f6d2423..ae20694f14c 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -990,40 +990,42 @@ struct UsingAliasKeyActions ) : left_alias_columns_keys(std::make_shared(left_plan_output_columns)) , right_alias_columns_keys(std::make_shared(right_plan_output_columns)) - {} - - void addLeftColumn(QueryTreeNodePtr & node, const ColumnsWithTypeAndName & plan_output_columns, const PlannerContextPtr & planner_context) { - addColumnImpl(left_alias_columns_keys, node, plan_output_columns, planner_context); + for (const auto * input_node : left_alias_columns_keys->getInputs()) + left_alias_columns_keys->addOrReplaceInOutputs(*input_node); + + for (const auto * input_node : right_alias_columns_keys->getInputs()) + right_alias_columns_keys->addOrReplaceInOutputs(*input_node); } - void addRightColumn(QueryTreeNodePtr & node, const ColumnsWithTypeAndName & plan_output_columns, const PlannerContextPtr & planner_context) + void addLeftColumn(QueryTreeNodePtr & node, const PlannerContextPtr & planner_context) { - addColumnImpl(right_alias_columns_keys, node, plan_output_columns, planner_context); + addColumnImpl(left_alias_columns_keys, node, planner_context); } - ActionsDAGPtr getLeftActions() + void addRightColumn(QueryTreeNodePtr & node, const PlannerContextPtr & planner_context) { - left_alias_columns_keys->projectInput(); - return std::move(left_alias_columns_keys); + addColumnImpl(right_alias_columns_keys, node, planner_context); } - ActionsDAGPtr getRightActions() - { - right_alias_columns_keys->projectInput(); - return std::move(right_alias_columns_keys); - } + ActionsDAGPtr getLeftActions() { return left_alias_columns_keys; } + ActionsDAGPtr getRightActions() { return right_alias_columns_keys; } private: - void addColumnImpl(ActionsDAGPtr & alias_columns_keys, QueryTreeNodePtr & node, const ColumnsWithTypeAndName & plan_output_columns, const PlannerContextPtr & planner_context) + + void addColumnImpl(ActionsDAGPtr & actions_dag, QueryTreeNodePtr & node, const PlannerContextPtr & planner_context) { - auto & column_node = node->as(); + auto & column_node = node->as(); if (column_node.hasExpression()) { - auto dag = buildActionsDAGFromExpressionNode(column_node.getExpressionOrThrow(), plan_output_columns, planner_context); + PlannerActionsVisitor actions_visitor(planner_context); + auto expression_dag_index_nodes = actions_visitor.visit(actions_dag, column_node.getExpressionOrThrow()); + for (const auto * out_node : expression_dag_index_nodes) + actions_dag->addOrReplaceInOutputs(*out_node); + const auto & left_inner_column_identifier = planner_context->getColumnNodeIdentifierOrThrow(node); - dag->addOrReplaceInOutputs(dag->addAlias(*dag->getOutputs().front(), left_inner_column_identifier)); - alias_columns_keys->mergeInplace(std::move(*dag)); + const auto & alias_output = actions_dag->addAlias(*actions_dag->getOutputs().back(), left_inner_column_identifier); + actions_dag->addOrReplaceInOutputs(alias_output); } } @@ -1110,12 +1112,12 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ auto & left_inner_column_node = inner_columns_list.getNodes().at(0); auto & left_inner_column = left_inner_column_node->as(); - using_alias_key_actions.addLeftColumn(left_inner_column_node, left_plan_output_columns, planner_context); + using_alias_key_actions.addLeftColumn(left_inner_column_node, planner_context); auto & right_inner_column_node = inner_columns_list.getNodes().at(1); auto & right_inner_column = right_inner_column_node->as(); - using_alias_key_actions.addRightColumn(right_inner_column_node, right_plan_output_columns, planner_context); + using_alias_key_actions.addRightColumn(right_inner_column_node, planner_context); const auto & join_node_using_column_node_type = join_node_using_column_node.getColumnType(); if (!left_inner_column.getColumnType()->equals(*join_node_using_column_node_type)) diff --git a/tests/queries/0_stateless/00722_inner_join.reference b/tests/queries/0_stateless/00722_inner_join.reference index b5e8a77a20d..c7ba26e3e6a 100644 --- a/tests/queries/0_stateless/00722_inner_join.reference +++ b/tests/queries/0_stateless/00722_inner_join.reference @@ -31,9 +31,9 @@ ┌─database─┬─name─┐ │ system │ one │ └──────────┴──────┘ -┌─database─┬─name─┐ -│ system │ one │ -└──────────┴──────┘ +┌─database─┬─t.name─┐ +│ system │ one │ +└──────────┴────────┘ 2 2 2 diff --git a/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference b/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference index a5a71560d00..bbeaae0d31b 100644 --- a/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference +++ b/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference @@ -52,27 +52,27 @@ Expression ((Project names + (Projection + DROP unused columns after JOIN))) Header: a2 String d2 String Join (JOIN FillRightFirst) - Header: __table1.a2 String - __table1.k UInt64 + Header: __table1.k UInt64 + __table1.a2 String __table4.d2 String Expression ((Actions for left table alias column keys + DROP unused columns after JOIN)) - Header: __table1.a2 String - __table1.k UInt64 + Header: __table1.k UInt64 + __table1.a2 String Join (JOIN FillRightFirst) - Header: __table1.a2 String - __table1.k UInt64 + Header: __table1.k UInt64 + __table1.a2 String Expression ((Actions for left table alias column keys + DROP unused columns after JOIN)) - Header: __table1.a2 String - __table1.k UInt64 + Header: __table1.k UInt64 + __table1.a2 String Join (JOIN FillRightFirst) - Header: __table1.a2 String - __table1.k UInt64 + Header: __table1.k UInt64 + __table1.a2 String Expression ((Actions for left table alias column keys + Change column names to column identifiers)) - Header: __table1.a2 String - __table1.k UInt64 + Header: __table1.k UInt64 + __table1.a2 String ReadFromMemoryStorage - Header: a2 String - k UInt64 + Header: k UInt64 + a2 String Expression ((Actions for right table alias column keys + Change column names to column identifiers)) Header: __table2.k UInt64 ReadFromMemoryStorage @@ -82,11 +82,11 @@ Header: a2 String ReadFromMemoryStorage Header: k UInt64 Expression ((Actions for right table alias column keys + Change column names to column identifiers)) - Header: __table4.d2 String - __table4.k UInt64 + Header: __table4.k UInt64 + __table4.d2 String ReadFromMemoryStorage - Header: d2 String - k UInt64 + Header: k UInt64 + d2 String EXPLAIN PLAN header = 1 SELECT b.bx FROM a JOIN (SELECT b1, b2 || 'x' AS bx FROM b ) AS b ON b.b1 = a.a1 diff --git a/tests/queries/0_stateless/02722_matcher_join_use_nulls.reference b/tests/queries/0_stateless/02722_matcher_join_use_nulls.reference index 746d02dc381..5617854bfd1 100644 --- a/tests/queries/0_stateless/02722_matcher_join_use_nulls.reference +++ b/tests/queries/0_stateless/02722_matcher_join_use_nulls.reference @@ -72,7 +72,7 @@ FROM ( LEFT JOIN (SELECT 2 :: UInt32 as a) t2 USING (a) ) ORDER BY 1; -1 Int32 +1 Int64 SELECT *, * APPLY toTypeName FROM ( SELECT t2.* @@ -80,7 +80,7 @@ FROM ( LEFT JOIN (SELECT 2 :: UInt32 as a) t2 USING (a) ) ORDER BY 1; -\N Nullable(UInt32) +\N Nullable(Int64) SELECT *, * APPLY toTypeName FROM ( SELECT * @@ -209,7 +209,7 @@ FROM ( RIGHT JOIN (SELECT 2 :: UInt32 as a) t2 USING (a) ) ORDER BY 1; -2 Nullable(Int32) +2 Nullable(Int64) SELECT *, * APPLY toTypeName FROM ( SELECT t2.* @@ -217,7 +217,7 @@ FROM ( RIGHT JOIN (SELECT 2 :: UInt32 as a) t2 USING (a) ) ORDER BY 1; -2 UInt32 +2 Int64 SELECT *, * APPLY toTypeName FROM ( SELECT * @@ -354,8 +354,8 @@ FROM ( FULL JOIN (SELECT 2 :: UInt32 as a) t2 USING (a) ) ORDER BY 1; -1 Nullable(Int32) -2 Nullable(Int32) +1 Nullable(Int64) +2 Nullable(Int64) SELECT *, * APPLY toTypeName FROM ( SELECT t2.* @@ -363,8 +363,8 @@ FROM ( FULL JOIN (SELECT 2 :: UInt32 as a) t2 USING (a) ) ORDER BY 1; -2 Nullable(UInt32) -\N Nullable(UInt32) +2 Nullable(Int64) +\N Nullable(Int64) SELECT *, * APPLY toTypeName FROM ( SELECT * diff --git a/tests/queries/0_stateless/02835_join_step_explain.reference b/tests/queries/0_stateless/02835_join_step_explain.reference index 06f4a9cfc99..d2042b56f55 100644 --- a/tests/queries/0_stateless/02835_join_step_explain.reference +++ b/tests/queries/0_stateless/02835_join_step_explain.reference @@ -56,20 +56,20 @@ Header: id UInt64 rhs.id UInt64 rhs.value_1 String Actions: INPUT : 0 -> __table1.id UInt64 : 0 - INPUT : 1 -> __table1.value_1 String : 1 - INPUT :: 2 -> __table1.value_2 UInt64 : 2 + INPUT :: 1 -> __table1.value_2 UInt64 : 1 + INPUT : 2 -> __table1.value_1 String : 2 INPUT : 3 -> __table2.value_1 String : 3 INPUT :: 4 -> __table2.value_2 UInt64 : 4 INPUT : 5 -> __table2.id UInt64 : 5 ALIAS __table1.id :: 0 -> id UInt64 : 6 - ALIAS __table1.value_1 :: 1 -> value_1 String : 0 - ALIAS __table2.value_1 :: 3 -> rhs.value_1 String : 1 + ALIAS __table1.value_1 :: 2 -> value_1 String : 0 + ALIAS __table2.value_1 :: 3 -> rhs.value_1 String : 2 ALIAS __table2.id :: 5 -> rhs.id UInt64 : 3 -Positions: 6 0 3 1 +Positions: 6 0 3 2 Join (JOIN FillRightFirst) Header: __table1.id UInt64 - __table1.value_1 String __table1.value_2 UInt64 + __table1.value_1 String __table2.value_1 String __table2.value_2 UInt64 __table2.id UInt64 @@ -80,37 +80,37 @@ Positions: 6 0 3 1 Clauses: [(__table1.id, __table1.value_2) = (__table2.id, __table2.value_2)] Expression ((JOIN actions + Change column names to column identifiers)) Header: __table1.id UInt64 - __table1.value_1 String __table1.value_2 UInt64 + __table1.value_1 String Actions: INPUT : 0 -> id UInt64 : 0 - INPUT : 1 -> value_1 String : 1 - INPUT : 2 -> value_2 UInt64 : 2 + INPUT : 1 -> value_2 UInt64 : 1 + INPUT : 2 -> value_1 String : 2 ALIAS id :: 0 -> __table1.id UInt64 : 3 - ALIAS value_1 :: 1 -> __table1.value_1 String : 0 - ALIAS value_2 :: 2 -> __table1.value_2 UInt64 : 1 + ALIAS value_2 :: 1 -> __table1.value_2 UInt64 : 0 + ALIAS value_1 :: 2 -> __table1.value_1 String : 1 Positions: 3 0 1 ReadFromMergeTree (default.test_table_1) Header: id UInt64 - value_1 String value_2 UInt64 + value_1 String ReadType: Default Parts: 1 Granules: 1 Expression ((JOIN actions + Change column names to column identifiers)) Header: __table2.id UInt64 - __table2.value_1 String __table2.value_2 UInt64 + __table2.value_1 String Actions: INPUT : 0 -> id UInt64 : 0 - INPUT : 1 -> value_1 String : 1 - INPUT : 2 -> value_2 UInt64 : 2 + INPUT : 1 -> value_2 UInt64 : 1 + INPUT : 2 -> value_1 String : 2 ALIAS id :: 0 -> __table2.id UInt64 : 3 - ALIAS value_1 :: 1 -> __table2.value_1 String : 0 - ALIAS value_2 :: 2 -> __table2.value_2 UInt64 : 1 + ALIAS value_2 :: 1 -> __table2.value_2 UInt64 : 0 + ALIAS value_1 :: 2 -> __table2.value_1 String : 1 Positions: 3 0 1 ReadFromMergeTree (default.test_table_2) Header: id UInt64 - value_1 String value_2 UInt64 + value_1 String ReadType: Default Parts: 1 Granules: 1 diff --git a/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference b/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference index 6ed281c757a..f4d8a77ef7f 100644 --- a/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference +++ b/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference @@ -1,2 +1,81 @@ 1 1 +369 124 123 b +369 124 123 b +124 +3693 1231 a 1231 +3693 1232 1231 1231 a +a +-- {echoOn } +-- USING alias column contains default in old analyzer (but both queries below should have the same result) +SELECT y * 2, s || 'a' FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +738 ba +7386 aa +13332 a +SELECT y * 2, s || 'a' FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; +738 ba +7386 aa +13332 a +SELECT (1, *) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +(1,369,123,'b',124) +(1,3693,1231,'a',0) +(1,6666,0,'',48) +SELECT (1, *) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +(1,369,'b') +(1,3693,'a') +(1,6666,'') +SELECT (1, t1.*) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL; +(1,0,'') +(1,123,'b') +(1,1231,'a') +SELECT (1, t1.*) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; +(1,'',6666) +(1,'a',3693) +(1,'b',369) +SELECT (1, t1.*, t2.*) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL; +(1,0,'',6666,48) +(1,123,'b',369,124) +(1,1231,'a',0,0) +SELECT (1, t1.*, t2.*) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; +(1,'',0,6666) +(1,'a',3693,0) +(1,'b',369,369) +SELECT t1.z, t2.z, t3.z FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1; +1 0 43 +1 48 1 +124 124 1 +1232 0 1232 +SELECT * FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1; +126 0 0 42 +369 123 b 124 0 +3693 1231 a 0 1231 +6666 0 48 0 +SELECT t1.*, t2.*, t3.* FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1; +0 126 0 42 +0 6666 48 0 +123 b 369 124 0 +1231 a 3693 0 1231 +SELECT (1, t1.*, t2.*, t3.*) FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1 SETTINGS allow_experimental_analyzer = 1; +(1,0,'',126,0,42) +(1,0,'',6666,48,0) +(1,123,'b',369,124,0) +(1,1231,'a',3693,0,1231) +SELECT y FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +369 +3693 +6666 +SELECT y FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; +369 +3693 +6666 +SELECT s FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL; + +a +b +SELECT s FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; + +a +b +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; diff --git a/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql b/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql index 7983b43d7e5..f92598c7c32 100644 --- a/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql +++ b/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql @@ -1,12 +1,57 @@ -CREATE TABLE t1 (x Int16, y ALIAS x + x * 2) ENGINE=MergeTree() ORDER BY x; -CREATE TABLE t2 (y Int16, z Int16) ENGINE=MergeTree() ORDER BY y; +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; + +CREATE TABLE t1 (x Int16, y Int64 ALIAS x + x * 2, z ALIAS x + 1, s String) ENGINE=MergeTree() ORDER BY x; +CREATE TABLE t2 (y Int128, z Int16) ENGINE=MergeTree() ORDER BY y; + +CREATE TABLE t3 (x Int16, y Int64 ALIAS x + x * 2, z ALIAS x + 1) ENGINE=MergeTree() ORDER BY x; + +INSERT INTO t1 VALUES (1231, 'a'), (123, 'b'); -INSERT INTO t1 VALUES (1231), (123); INSERT INTO t2 VALUES (6666, 48); -INSERT INTO t2 VALUES (369, 50); +INSERT INTO t2 VALUES (369, 124); + +INSERT INTO t3 VALUES (1231), (42); SELECT count() FROM t1 INNER JOIN t2 USING (y); SELECT count() FROM t2 INNER JOIN t1 USING (y); +-- `SELECT *` works differently for ALIAS columns with analyzer +SELECT * FROM t1 INNER JOIN t2 USING (y, z) SETTINGS allow_experimental_analyzer = 1; +SELECT * FROM t2 INNER JOIN t1 USING (y, z) SETTINGS allow_experimental_analyzer = 1; +SELECT t2.z FROM t1 INNER JOIN t2 USING (y); + +SELECT * FROM t1 INNER JOIN t3 USING (y) SETTINGS allow_experimental_analyzer = 1; +SELECT * FROM t3 INNER JOIN t1 USING (y, z) SETTINGS allow_experimental_analyzer = 1; +SELECT s FROM t1 INNER JOIN t3 USING (y); + +-- {echoOn } +-- USING alias column contains default in old analyzer (but both queries below should have the same result) +SELECT y * 2, s || 'a' FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT y * 2, s || 'a' FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; + +SELECT (1, *) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT (1, *) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; + +SELECT (1, t1.*) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL; +SELECT (1, t1.*) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; + +SELECT (1, t1.*, t2.*) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL; +SELECT (1, t1.*, t2.*) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; + +SELECT t1.z, t2.z, t3.z FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1; +SELECT * FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1; +SELECT t1.*, t2.*, t3.* FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1; +SELECT (1, t1.*, t2.*, t3.*) FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1 SETTINGS allow_experimental_analyzer = 1; + +SELECT y FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT y FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; + +SELECT s FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL; +SELECT s FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; + + DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; diff --git a/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.reference b/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.reference new file mode 100644 index 00000000000..a5c8806279f --- /dev/null +++ b/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.reference @@ -0,0 +1,2 @@ +3 +3 diff --git a/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.sql b/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.sql new file mode 100644 index 00000000000..7101d5c89ae --- /dev/null +++ b/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.sql @@ -0,0 +1,31 @@ +SET join_use_nulls = 1; + +select c FROM ( + select + d2.c + from ( select 1 as a, 2 as b ) d1 + FULL join ( select 1 as a, 3 as c ) d2 + on (d1.a = d2.a) +) +; + +with d1 as ( + select + 1 as a, + 2 as b +), +d2 as ( + select + 1 as a, + 3 as c +), +joined as ( + select + d1.*, + d2.c + from d1 + inner join d2 + on (d1.a = d2.a) +) +select c +from joined; From 4f273c60c3432f1e919bae87188563e2a078cec1 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 20 Mar 2024 09:46:07 +0000 Subject: [PATCH 0303/1165] CI: disable some kerberos and hdfs integration tests for arm #ci_set_arm #batch_2 #no_merge_commit --- tests/integration/helpers/cluster.py | 7 +++++++ tests/integration/test_kerberos_auth/test.py | 5 ++++- tests/integration/test_storage_kerberized_hdfs/test.py | 9 ++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 152b0d066ee..b3088ba7c81 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1,8 +1,10 @@ import base64 import errno +from functools import cache import http.client import logging import os +import platform import stat import os.path as p import pprint @@ -4743,3 +4745,8 @@ class ClickHouseKiller(object): def __exit__(self, exc_type, exc_val, exc_tb): self.clickhouse_node.start_clickhouse() + + +@cache +def is_arm(): + return any(arch in platform.processor().lower() for arch in ("arm, aarch")) diff --git a/tests/integration/test_kerberos_auth/test.py b/tests/integration/test_kerberos_auth/test.py index a41255cff3e..0025ca7c6b6 100644 --- a/tests/integration/test_kerberos_auth/test.py +++ b/tests/integration/test_kerberos_auth/test.py @@ -1,5 +1,5 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm cluster = ClickHouseCluster(__file__) instance1 = cluster.add_instance( @@ -62,10 +62,12 @@ def make_auth(instance): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_kerberos_auth_with_keytab(kerberos_cluster): assert make_auth(instance1) == "kuser\n" +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_kerberos_auth_without_keytab(kerberos_cluster): assert ( "DB::Exception: : Authentication failed: password is incorrect, or there is no user with such name." @@ -73,6 +75,7 @@ def test_kerberos_auth_without_keytab(kerberos_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_bad_path_to_keytab(kerberos_cluster): assert ( "DB::Exception: : Authentication failed: password is incorrect, or there is no user with such name." diff --git a/tests/integration/test_storage_kerberized_hdfs/test.py b/tests/integration/test_storage_kerberized_hdfs/test.py index 5ac8b4670f9..c33a9614945 100644 --- a/tests/integration/test_storage_kerberized_hdfs/test.py +++ b/tests/integration/test_storage_kerberized_hdfs/test.py @@ -3,7 +3,7 @@ import pytest import os -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm import subprocess cluster = ClickHouseCluster(__file__) @@ -29,6 +29,7 @@ def started_cluster(): cluster.shutdown() +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_table(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -44,6 +45,7 @@ def test_read_table(started_cluster): assert select_read == data +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_write_storage(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -59,6 +61,7 @@ def test_read_write_storage(started_cluster): assert select_read == "1\tMark\t72.53\n" +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_write_storage_not_expired(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -76,6 +79,7 @@ def test_write_storage_not_expired(started_cluster): assert select_read == "1\tMark\t72.53\n" +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_two_users(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -98,6 +102,7 @@ def test_two_users(started_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_table_expired(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -118,6 +123,7 @@ def test_read_table_expired(started_cluster): started_cluster.unpause_container("hdfskerberos") +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_prohibited(started_cluster): node1.query( "create table HDFSStorTwoProhibited (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/storage_user_two_prohibited', 'TSV')" @@ -132,6 +138,7 @@ def test_prohibited(started_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_cache_path(started_cluster): node1.query( "create table HDFSStorCachePath (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://dedicatedcachepath@kerberizedhdfs1:9010/storage_dedicated_cache_path', 'TSV')" From 7b8c5f35b522c23a9abae735739496622cdef703 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 20 Mar 2024 14:05:06 +0000 Subject: [PATCH 0304/1165] disable test_strage_hdfs module #ci_set_arm #batch_2 #no_merge_commit --- tests/integration/test_storage_hdfs/test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 121263fb622..9dec1954406 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -2,10 +2,13 @@ import os import pytest import time -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV from pyhdfs import HdfsClient +if is_arm(): + pytestmark = pytest.mark.skip + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", From 984c7e69a0a58dfe1930a9e0ee18e8f19f52e9d1 Mon Sep 17 00:00:00 2001 From: Max K Date: Fri, 15 Mar 2024 15:18:45 +0100 Subject: [PATCH 0305/1165] disable test_allowed_url_with_config tests with HDFS #no_merge_commit #ci_set_arm --- tests/ci/ci_config.py | 2 +- .../test_allowed_url_from_config/test.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 7c213da27ec..d2caf75f4bd 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -380,7 +380,7 @@ upgrade_check_digest = DigestConfig( docker=["clickhouse/upgrade-check"], ) integration_check_digest = DigestConfig( - include_paths=["./tests/ci/integration_test_check.py", "./tests/integration"], + include_paths=["./tests/ci/integration_test_check.py", "./tests/integration/"], exclude_files=[".md"], docker=IMAGES.copy(), ) diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 3106cf12702..df8934aa69b 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -1,5 +1,5 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", main_configs=["configs/config_with_hosts.xml"]) @@ -16,9 +16,11 @@ node5 = cluster.add_instance( "node5", main_configs=["configs/config_without_allowed_hosts.xml"] ) node6 = cluster.add_instance("node6", main_configs=["configs/config_for_remote.xml"]) -node7 = cluster.add_instance( - "node7", main_configs=["configs/config_for_redirect.xml"], with_hdfs=True -) + +if not is_arm(): + node7 = cluster.add_instance( + "node7", main_configs=["configs/config_for_redirect.xml"], with_hdfs=True + ) @pytest.fixture(scope="module") @@ -270,6 +272,7 @@ def test_table_function_remote(start_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_redirect(start_cluster): hdfs_api = start_cluster.hdfs_api @@ -284,6 +287,7 @@ def test_redirect(start_cluster): node7.query("DROP TABLE table_test_7_1") +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_HDFS(start_cluster): assert "not allowed" in node7.query_and_get_error( "CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')" @@ -293,6 +297,7 @@ def test_HDFS(start_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_schema_inference(start_cluster): error = node7.query_and_get_error("desc url('http://test.com`, 'TSVRaw'')") assert error.find("ReadWriteBufferFromHTTPBase") == -1 From 55737de0e604760fadeacc5aae82b54d69d52df7 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 20 Mar 2024 15:48:53 +0000 Subject: [PATCH 0306/1165] disable more hdfs and kafka inttests #ci_set_arm --- tests/integration/test_disk_types/test.py | 9 ++++++--- .../test_endpoint_macro_substitution/test.py | 4 +++- tests/integration/test_format_avro_confluent/test.py | 6 +++++- tests/integration/test_jdbc_bridge/test.py | 5 ++++- tests/integration/test_kafka_bad_messages/test.py | 6 +++++- tests/integration/test_keeper_disks/test.py | 6 +++++- tests/integration/test_log_family_hdfs/test.py | 5 ++++- .../test_materialized_mysql_database/test.py | 5 +++++ tests/integration/test_merge_tree_hdfs/test.py | 6 +++++- tests/integration/test_mysql_protocol/test.py | 11 ++++++++++- tests/integration/test_redirect_url_storage/test.py | 7 ++++++- tests/integration/test_storage_kafka/test.py | 4 +++- 12 files changed, 61 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index b9b8ef2010d..3c4169be4de 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -1,14 +1,17 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV disk_types = { "default": "Local", "disk_s3": "S3", - "disk_hdfs": "HDFS", "disk_encrypted": "S3", } +# do not test HDFS on ARM +if not is_arm(): + disk_types["disk_hdfs"] = "HDFS" + @pytest.fixture(scope="module") def cluster(): @@ -18,7 +21,7 @@ def cluster(): "node", main_configs=["configs/storage.xml"], with_minio=True, - with_hdfs=True, + with_hdfs=not is_arm(), ) cluster.start() diff --git a/tests/integration/test_endpoint_macro_substitution/test.py b/tests/integration/test_endpoint_macro_substitution/test.py index ee72fb9b492..22a649e2225 100644 --- a/tests/integration/test_endpoint_macro_substitution/test.py +++ b/tests/integration/test_endpoint_macro_substitution/test.py @@ -1,5 +1,5 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV from pyhdfs import HdfsClient @@ -10,6 +10,8 @@ disk_types = { "disk_encrypted": "S3", } +if is_arm(): + pytestmark = pytest.mark.skip @pytest.fixture(scope="module") def cluster(): diff --git a/tests/integration/test_format_avro_confluent/test.py b/tests/integration/test_format_avro_confluent/test.py index 540f90ae05e..ccaaee83514 100644 --- a/tests/integration/test_format_avro_confluent/test.py +++ b/tests/integration/test_format_avro_confluent/test.py @@ -8,9 +8,13 @@ from confluent_kafka.avro.cached_schema_registry_client import ( CachedSchemaRegistryClient, ) from confluent_kafka.avro.serializer.message_serializer import MessageSerializer -from helpers.cluster import ClickHouseCluster, ClickHouseInstance +from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm from urllib import parse +# Skip on ARM due to Confluent/Kafka +if is_arm(): + pytestmark = pytest.mark.skip + @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_jdbc_bridge/test.py b/tests/integration/test_jdbc_bridge/test.py index 0e41cc8c8b7..c4a0a525df3 100644 --- a/tests/integration/test_jdbc_bridge/test.py +++ b/tests/integration/test_jdbc_bridge/test.py @@ -3,7 +3,7 @@ import os.path as p import pytest import uuid -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV from string import Template @@ -14,6 +14,9 @@ instance = cluster.add_instance( datasource = "self" records = 1000 +if is_arm(): + pytestmark = pytest.mark.skip + @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_kafka_bad_messages/test.py b/tests/integration/test_kafka_bad_messages/test.py index 954b6042305..0446ca5cb47 100644 --- a/tests/integration/test_kafka_bad_messages/test.py +++ b/tests/integration/test_kafka_bad_messages/test.py @@ -2,10 +2,14 @@ import time import logging import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection from kafka.admin import NewTopic +if is_arm(): + pytestmark = pytest.mark.skip + + cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( "instance", diff --git a/tests/integration/test_keeper_disks/test.py b/tests/integration/test_keeper_disks/test.py index e41837b89b4..0c91aa03419 100644 --- a/tests/integration/test_keeper_disks/test.py +++ b/tests/integration/test_keeper_disks/test.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm import helpers.keeper_utils as keeper_utils from minio.deleteobjects import DeleteObject import os +if is_arm(): + pytestmark = pytest.mark.skip + + CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) cluster = ClickHouseCluster(__file__) node_logs = cluster.add_instance( diff --git a/tests/integration/test_log_family_hdfs/test.py b/tests/integration/test_log_family_hdfs/test.py index e8afe364ec4..6c3d28d2e3c 100644 --- a/tests/integration/test_log_family_hdfs/test.py +++ b/tests/integration/test_log_family_hdfs/test.py @@ -2,10 +2,13 @@ import logging import sys import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from pyhdfs import HdfsClient +if is_arm(): + pytestmark = pytest.mark.skip + @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_materialized_mysql_database/test.py b/tests/integration/test_materialized_mysql_database/test.py index 89c69c42adc..57e496fe737 100644 --- a/tests/integration/test_materialized_mysql_database/test.py +++ b/tests/integration/test_materialized_mysql_database/test.py @@ -6,6 +6,7 @@ from helpers.cluster import ( ClickHouseCluster, ClickHouseInstance, get_docker_compose_path, + is_arm, ) import logging @@ -13,6 +14,10 @@ from . import materialized_with_ddl DOCKER_COMPOSE_PATH = get_docker_compose_path() +# skip all test on arm due to no arm support in mysql57 +if is_arm(): + pytestmark = pytest.mark.skip + cluster = ClickHouseCluster(__file__) mysql_node = None mysql8_node = None diff --git a/tests/integration/test_merge_tree_hdfs/test.py b/tests/integration/test_merge_tree_hdfs/test.py index 95b63a5c8a3..5ca7dc5feb0 100644 --- a/tests/integration/test_merge_tree_hdfs/test.py +++ b/tests/integration/test_merge_tree_hdfs/test.py @@ -3,7 +3,7 @@ import time import os import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.utility import generate_values from helpers.wait_for_helpers import wait_for_delete_inactive_parts from helpers.wait_for_helpers import wait_for_delete_empty_parts @@ -16,6 +16,10 @@ CONFIG_PATH = os.path.join( ) +if is_arm(): + pytestmark = pytest.mark.skip + + def create_table(cluster, table_name, additional_settings=None): node = cluster.instances["node"] diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py index 7a69d07633c..e641d4c2300 100644 --- a/tests/integration/test_mysql_protocol/test.py +++ b/tests/integration/test_mysql_protocol/test.py @@ -12,11 +12,20 @@ from typing import Literal import docker import pymysql.connections import pytest -from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check +from helpers.cluster import ( + ClickHouseCluster, + get_docker_compose_path, + is_arm, + run_and_check, +) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() +if is_arm(): + pytestmark = pytest.mark.skip + + cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", diff --git a/tests/integration/test_redirect_url_storage/test.py b/tests/integration/test_redirect_url_storage/test.py index 033f02d7bde..ba3fb3e14ab 100644 --- a/tests/integration/test_redirect_url_storage/test.py +++ b/tests/integration/test_redirect_url_storage/test.py @@ -1,10 +1,15 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.network import PartitionManager import threading import time +# skip all tests in the module on ARM due to HDFS +if is_arm(): + pytestmark = pytest.mark.skip + + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index dea1ea49851..5bdea179449 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -22,7 +22,7 @@ import kafka.errors import pytest from google.protobuf.internal.encoder import _VarintBytes from helpers.client import QueryRuntimeException -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.network import PartitionManager from helpers.test_tools import TSV from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection @@ -40,6 +40,8 @@ from . import kafka_pb2 from . import social_pb2 from . import message_with_repeated_pb2 +if is_arm(): + pytestmark = pytest.mark.skip # TODO: add test for run-time offset update in CH, if we manually update it on Kafka side. # TODO: add test for SELECT LIMIT is working. From 9f3c625ab5034cadb2805bcd2068f4341eab5829 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 20 Mar 2024 17:23:08 +0000 Subject: [PATCH 0307/1165] fix mysql client tests to use mysql8 image #ci_set_integration #ci_set_arm --- .../compose/docker_compose_mysql_client.yml | 2 +- .../test_endpoint_macro_substitution/test.py | 1 + tests/integration/test_kerberos_auth/test.py | 7 ++++--- tests/integration/test_mysql_protocol/test.py | 4 ---- tests/integration/test_storage_kerberized_hdfs/test.py | 10 +++------- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/integration/compose/docker_compose_mysql_client.yml b/tests/integration/compose/docker_compose_mysql_client.yml index 5b37b6e6c09..ee590118d4f 100644 --- a/tests/integration/compose/docker_compose_mysql_client.yml +++ b/tests/integration/compose/docker_compose_mysql_client.yml @@ -1,7 +1,7 @@ version: '2.3' services: mysql_client: - image: mysql:5.7 + image: mysql:8.0 restart: always environment: MYSQL_ALLOW_EMPTY_PASSWORD: 1 diff --git a/tests/integration/test_endpoint_macro_substitution/test.py b/tests/integration/test_endpoint_macro_substitution/test.py index 22a649e2225..7dc282a980f 100644 --- a/tests/integration/test_endpoint_macro_substitution/test.py +++ b/tests/integration/test_endpoint_macro_substitution/test.py @@ -13,6 +13,7 @@ disk_types = { if is_arm(): pytestmark = pytest.mark.skip + @pytest.fixture(scope="module") def cluster(): try: diff --git a/tests/integration/test_kerberos_auth/test.py b/tests/integration/test_kerberos_auth/test.py index 0025ca7c6b6..2d0dce33c14 100644 --- a/tests/integration/test_kerberos_auth/test.py +++ b/tests/integration/test_kerberos_auth/test.py @@ -1,6 +1,10 @@ import pytest from helpers.cluster import ClickHouseCluster, is_arm +if is_arm(): + pytestmark = pytest.mark.skip + + cluster = ClickHouseCluster(__file__) instance1 = cluster.add_instance( "instance1", @@ -62,12 +66,10 @@ def make_auth(instance): ) -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_kerberos_auth_with_keytab(kerberos_cluster): assert make_auth(instance1) == "kuser\n" -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_kerberos_auth_without_keytab(kerberos_cluster): assert ( "DB::Exception: : Authentication failed: password is incorrect, or there is no user with such name." @@ -75,7 +77,6 @@ def test_kerberos_auth_without_keytab(kerberos_cluster): ) -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_bad_path_to_keytab(kerberos_cluster): assert ( "DB::Exception: : Authentication failed: password is incorrect, or there is no user with such name." diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py index e641d4c2300..094ae7b9fd0 100644 --- a/tests/integration/test_mysql_protocol/test.py +++ b/tests/integration/test_mysql_protocol/test.py @@ -15,16 +15,12 @@ import pytest from helpers.cluster import ( ClickHouseCluster, get_docker_compose_path, - is_arm, run_and_check, ) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() -if is_arm(): - pytestmark = pytest.mark.skip - cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_storage_kerberized_hdfs/test.py b/tests/integration/test_storage_kerberized_hdfs/test.py index c33a9614945..c72152fa376 100644 --- a/tests/integration/test_storage_kerberized_hdfs/test.py +++ b/tests/integration/test_storage_kerberized_hdfs/test.py @@ -6,6 +6,9 @@ import os from helpers.cluster import ClickHouseCluster, is_arm import subprocess +if is_arm(): + pytestmark = pytest.mark.skip + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", @@ -29,7 +32,6 @@ def started_cluster(): cluster.shutdown() -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_table(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -45,7 +47,6 @@ def test_read_table(started_cluster): assert select_read == data -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_write_storage(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -61,7 +62,6 @@ def test_read_write_storage(started_cluster): assert select_read == "1\tMark\t72.53\n" -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_write_storage_not_expired(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -79,7 +79,6 @@ def test_write_storage_not_expired(started_cluster): assert select_read == "1\tMark\t72.53\n" -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_two_users(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -102,7 +101,6 @@ def test_two_users(started_cluster): ) -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_table_expired(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -123,7 +121,6 @@ def test_read_table_expired(started_cluster): started_cluster.unpause_container("hdfskerberos") -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_prohibited(started_cluster): node1.query( "create table HDFSStorTwoProhibited (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/storage_user_two_prohibited', 'TSV')" @@ -138,7 +135,6 @@ def test_prohibited(started_cluster): ) -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_cache_path(started_cluster): node1.query( "create table HDFSStorCachePath (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://dedicatedcachepath@kerberizedhdfs1:9010/storage_dedicated_cache_path', 'TSV')" From f44127ce19f3723eac91fcc795d0adf0adb0b427 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 20 Mar 2024 19:13:51 +0100 Subject: [PATCH 0308/1165] fix --- src/Databases/DatabaseReplicatedWorker.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 0a6e8f9345e..51065062995 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -77,10 +77,7 @@ void DatabaseReplicatedDDLWorker::initializeReplication() zookeeper->deleteEphemeralNodeIfContentMatches(active_path, active_id); if (active_node_holder) active_node_holder->setAlreadyRemoved(); - zookeeper->create(active_path, active_id, zkutil::CreateMode::Ephemeral); active_node_holder.reset(); - active_node_holder_zookeeper = zookeeper; - active_node_holder = zkutil::EphemeralNodeHolder::existing(active_path, *active_node_holder_zookeeper); String log_ptr_str = zookeeper->get(database->replica_path + "/log_ptr"); UInt32 our_log_ptr = parse(log_ptr_str); @@ -129,9 +126,15 @@ void DatabaseReplicatedDDLWorker::initializeReplication() initializeLogPointer(log_entry_name); } - std::lock_guard lock{database->metadata_mutex}; - if (!database->checkDigestValid(context, false)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Inconsistent database metadata after reconnection to ZooKeeper"); + { + std::lock_guard lock{database->metadata_mutex}; + if (!database->checkDigestValid(context, false)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Inconsistent database metadata after reconnection to ZooKeeper"); + } + + zookeeper->create(active_path, active_id, zkutil::CreateMode::Ephemeral); + active_node_holder_zookeeper = zookeeper; + active_node_holder = zkutil::EphemeralNodeHolder::existing(active_path, *active_node_holder_zookeeper); } String DatabaseReplicatedDDLWorker::enqueueQuery(DDLLogEntry & entry) From d190ee8113ae6eae7d77850e884fae7e56c65924 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 20 Mar 2024 19:19:30 +0100 Subject: [PATCH 0309/1165] Update AsynchronousInsertQueue.h --- src/Interpreters/AsynchronousInsertQueue.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 1a4678e3a7d..a8ee064877d 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -63,6 +63,8 @@ public: PushResult pushQueryWithBlock(ASTPtr query, Block block, ContextPtr query_context); size_t getPoolSize() const { return pool_size; } + /// This method should be called manually because it's not flushed automatically in dtor + /// because all tables may be already unloaded when we destroy AsynchronousInsertQueue void flushAndShutdown(); private: From 040b191899c8dab75ebc0c8149a170b177264a46 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 20 Mar 2024 20:25:38 +0100 Subject: [PATCH 0310/1165] Fix readBigAt --- src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 13f62c7ac25..5941dca0b48 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -279,26 +279,24 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran sleep_time_with_backoff_milliseconds *= 2; }; - Azure::Storage::Blobs::DownloadBlobOptions download_options; - download_options.Range = {static_cast(range_begin), range_begin+n}; - for (size_t i = 0; i < max_single_download_retries; ++i) + + for (size_t i = 0; i < max_single_download_retries && n > 0; ++i) { size_t bytes_copied = 0; try { + Azure::Storage::Blobs::DownloadBlobOptions download_options; + download_options.Range = {static_cast(range_begin), n}; auto download_response = blob_client->Download(download_options); + std::unique_ptr body_stream = std::move(download_response.Value.BodyStream); - auto length = body_stream->Length(); - char buffer[length]; - body_stream->Read(reinterpret_cast(buffer), length); - std::istringstream string_stream(String(static_cast(buffer),length)); // STYLE_CHECK_ALLOW_STD_STRING_STREAM + auto bytes = body_stream->ReadToCount(reinterpret_cast(data_ptr), body_stream->Length()); + std::istringstream string_stream(String(static_cast(data_ptr),bytes)); // STYLE_CHECK_ALLOW_STD_STRING_STREAM copyFromIStreamWithProgressCallback(string_stream, to, n, progress_callback, &bytes_copied); if (read_settings.remote_throttler) read_settings.remote_throttler->add(bytes_copied, ProfileEvents::RemoteReadThrottlerBytes, ProfileEvents::RemoteReadThrottlerSleepMicroseconds); - - break; } catch (const Azure::Core::RequestFailedException & e) { From e9bc922d947eff6afe94273b4e0dff748fac39be Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 20 Mar 2024 12:52:03 +0100 Subject: [PATCH 0311/1165] Fix SLRU case, not yet working --- src/Interpreters/Cache/EvictionCandidates.h | 2 +- src/Interpreters/Cache/FileCache.cpp | 43 ++- src/Interpreters/Cache/IFileCachePriority.cpp | 9 + src/Interpreters/Cache/IFileCachePriority.h | 49 +++- .../Cache/LRUFileCachePriority.cpp | 80 ++++-- src/Interpreters/Cache/LRUFileCachePriority.h | 20 +- .../Cache/SLRUFileCachePriority.cpp | 267 ++++++++++++++---- .../Cache/SLRUFileCachePriority.h | 17 +- 8 files changed, 377 insertions(+), 110 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index 19df0453971..43333041f84 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -22,7 +22,7 @@ public: auto end() const { return candidates.end(); } using FinalizeEvictionFunc = std::function; - void setFinalizeEvictionFunc(FinalizeEvictionFunc && func) { finalize_eviction_func = func; } + void setFinalizeEvictionFunc(FinalizeEvictionFunc && func) { finalize_eviction_func = std::move(func); } private: struct KeyCandidates diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 1e03f6f9fd8..5a3551b1254 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -839,10 +839,15 @@ bool FileCache::tryReserve( if (query_priority) { if (!query_priority->collectCandidatesForEviction( - size, reserve_stat, eviction_candidates, {}, user.user_id, reached_size_limit, reached_elements_limit, cache_lock)) + size, reserve_stat, eviction_candidates, {}, user.user_id, + reached_size_limit, reached_elements_limit, cache_lock)) + { return false; + } + + LOG_TEST(log, "Query limits satisfied (while reserving for {}:{})", + file_segment.key(), file_segment.offset()); - LOG_TEST(log, "Query limits satisfied (while reserving for {}:{})", file_segment.key(), file_segment.offset()); /// If we have enough space in query_priority, we are not interested about stat there anymore. /// Clean the stat before iterating main_priority to avoid calculating any segment stat twice. reserve_stat.stat_by_kind.clear(); @@ -853,8 +858,11 @@ bool FileCache::tryReserve( chassert(!queue_iterator || file_segment.getReservedSize() > 0); if (!main_priority->collectCandidatesForEviction( - size, reserve_stat, eviction_candidates, queue_iterator, user.user_id, reached_size_limit, reached_elements_limit, cache_lock)) + size, reserve_stat, eviction_candidates, queue_iterator, user.user_id, + reached_size_limit, reached_elements_limit, cache_lock)) + { return false; + } if (!file_segment.getKeyMetadata()->createBaseDirectory()) return false; @@ -875,8 +883,20 @@ bool FileCache::tryReserve( /// If we reached the elements limit - we will evict at least 1 element, /// then we do not need to hold anything, otherwise (if we reached limit only by size) /// we will also evict at least one element, so hold elements count is awlays zero here. - auto hold_space = std::make_unique( - hold_size, /* hold_elements */0, queue_iterator, *main_priority, cache_lock); + + std::unique_ptr hold_space; + if (hold_size) + { + const auto queue_entry_type = queue_iterator + ? queue_iterator->getType() + : main_priority->getDefaultQueueEntryType(); + + hold_space = std::make_unique( + hold_size, /* hold_elements */0, queue_entry_type, *main_priority, cache_lock); + } + + LOG_TEST(log, "Eviction candidates: {}, hold space: {}. {}", + eviction_candidates.size(), hold_size, main_priority->getStateInfoForLog(cache_lock)); cache_lock.unlock(); try @@ -893,9 +913,18 @@ bool FileCache::tryReserve( } cache_lock.lock(); + /// Invalidate and remove queue entries and execute (only for SLRU) finalize func. eviction_candidates.finalize(query_context.get(), cache_lock); } + else if (!main_priority->canFit(size, /* elements */queue_iterator ? 0 : 1, cache_lock, queue_iterator)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot fit {} in cache, but collection of eviction candidates succeeded. " + "This is a bug. Queue entry type: {}. Cache info: {}", + size, queue_iterator ? queue_iterator->getType() : FileCacheQueueEntryType::None, + main_priority->getStateInfoForLog(cache_lock)); + } /// Space reservation is incremental, so file_segment_metadata is created first (with state Empty), /// and queue_iterator is assigned on first space reservation attempt @@ -904,14 +933,18 @@ bool FileCache::tryReserve( { /// Increase size of queue entry. queue_iterator->incrementSize(size, cache_lock); + main_priority->check(cache_lock); } else { /// Create a new queue entry and assign currently reserved size to it. queue_iterator = main_priority->add(file_segment.getKeyMetadata(), file_segment.offset(), size, user, cache_lock); file_segment.setQueueIterator(queue_iterator); + main_priority->check(cache_lock); } + main_priority->check(cache_lock); + if (query_context) { auto query_queue_it = query_context->tryGet(file_segment.key(), file_segment.offset(), cache_lock); diff --git a/src/Interpreters/Cache/IFileCachePriority.cpp b/src/Interpreters/Cache/IFileCachePriority.cpp index eb396a1e323..9ef245dd8d2 100644 --- a/src/Interpreters/Cache/IFileCachePriority.cpp +++ b/src/Interpreters/Cache/IFileCachePriority.cpp @@ -37,4 +37,13 @@ IFileCachePriority::Entry::Entry(const Entry & other) { } +void IFileCachePriority::check(const CachePriorityGuard::Lock & lock) const +{ + if (getSize(lock) > max_size || getElementsCount(lock) > max_elements) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache limits violated. " + "{}", getStateInfoForLog(lock)); + } +} + } diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index f275dfa485a..ddca9d84983 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -39,6 +39,8 @@ public: std::atomic size; size_t hits = 0; + std::string toString() const { return fmt::format("{}:{}:{}", key, offset, size); } + bool isEvicting(const CachePriorityGuard::Lock &) const { return evicting; } bool isEvicting(const LockedKey &) const { return evicting; } /// This does not look good to have isEvicting with two options for locks, @@ -103,6 +105,12 @@ public: virtual size_t getElementsCountApprox() const = 0; + virtual QueueEntryType getDefaultQueueEntryType() const = 0; + + virtual std::string getStateInfoForLog(const CachePriorityGuard::Lock &) const = 0; + + virtual void check(const CachePriorityGuard::Lock &) const; + /// Throws exception if there is not enough size to fit it. virtual IteratorPtr add( /// NOLINT KeyMetadataPtr key_metadata, @@ -144,31 +152,52 @@ public: virtual bool modifySizeLimits(size_t max_size_, size_t max_elements_, double size_ratio_, const CachePriorityGuard::Lock &) = 0; - struct HoldSpace : boost::noncopyable + struct HoldSpace : private boost::noncopyable { - HoldSpace(size_t size_, size_t elements_, IteratorPtr reservee_, IFileCachePriority & priority_, const CachePriorityGuard::Lock & lock) - : size(size_), elements(elements_), reservee(reservee_), priority(priority_) + HoldSpace( + size_t size_, + size_t elements_, + QueueEntryType queue_entry_type_, + IFileCachePriority & priority_, + const CachePriorityGuard::Lock & lock) + : size(size_), elements(elements_), queue_entry_type(queue_entry_type_), priority(priority_) { - priority.holdImpl(size, elements, reservee, lock); + priority.holdImpl(size, elements, queue_entry_type, lock); + } + + void release() + { + if (released) + return; + released = true; + priority.releaseImpl(size, elements, queue_entry_type); } ~HoldSpace() { - priority.releaseImpl(size, elements, reservee); + if (!released) + release(); } - size_t size; - size_t elements; - IteratorPtr reservee; + private: + const size_t size; + const size_t elements; + const QueueEntryType queue_entry_type; IFileCachePriority & priority; + bool released = false; }; HoldSpace takeHold(); protected: IFileCachePriority(size_t max_size_, size_t max_elements_); - virtual void holdImpl(size_t size, size_t elements, IteratorPtr reservee, const CachePriorityGuard::Lock & lock) = 0; - virtual void releaseImpl(size_t size, size_t elements, IteratorPtr) = 0; + virtual void holdImpl( + size_t size, + size_t elements, + QueueEntryType queue_entry_type, + const CachePriorityGuard::Lock & lock) = 0; + + virtual void releaseImpl(size_t size, size_t elements, QueueEntryType queue_entry_type) = 0; size_t max_size = 0; size_t max_elements = 0; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 461818678c1..04622d54a17 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -30,8 +30,13 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -LRUFileCachePriority::LRUFileCachePriority(size_t max_size_, size_t max_elements_, StatePtr state_) +LRUFileCachePriority::LRUFileCachePriority( + size_t max_size_, + size_t max_elements_, + StatePtr state_, + const std::string & description_) : IFileCachePriority(max_size_, max_elements_) + , description(description_) { if (state_) state = state_; @@ -65,21 +70,21 @@ LRUFileCachePriority::LRUIterator LRUFileCachePriority::add(EntryPtr entry, cons { /// entry.size == 0 means entry was invalidated. if (queue_entry->size != 0 && queue_entry->key == entry->key && queue_entry->offset == entry->offset) + { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Attempt to add duplicate queue entry to queue. " - "(Key: {}, offset: {}, size: {})", - entry->key, entry->offset, entry->size); + "Attempt to add duplicate queue entry to queue: {}", + entry->toString()); + } } #endif - const auto & size_limit = getSizeLimit(lock); - if (size_limit && state->current_size + entry->size > size_limit) + if (!canFit(entry->size, 1, lock)) { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Not enough space to add {}:{} with size {}: current size: {}/{}", - entry->key, entry->offset, entry->size, state->current_size, size_limit); + "Not enough space to add a new entry {}. Current state: {}", + entry->toString(), getStateInfoForLog(lock)); } auto iterator = queue.insert(queue.end(), entry); @@ -94,7 +99,8 @@ LRUFileCachePriority::LRUIterator LRUFileCachePriority::add(EntryPtr entry, cons return LRUIterator(this, iterator); } -LRUFileCachePriority::LRUQueue::iterator LRUFileCachePriority::remove(LRUQueue::iterator it, const CachePriorityGuard::Lock &) +LRUFileCachePriority::LRUQueue::iterator +LRUFileCachePriority::remove(LRUQueue::iterator it, const CachePriorityGuard::Lock &) { /// If size is 0, entry is invalidated, current_elements_num was already updated. const auto & entry = **it; @@ -294,7 +300,8 @@ bool LRUFileCachePriority::collectCandidatesForEviction( return can_fit(); } -LRUFileCachePriority::LRUIterator LRUFileCachePriority::move(LRUIterator & it, LRUFileCachePriority & other, const CachePriorityGuard::Lock &) +LRUFileCachePriority::LRUIterator +LRUFileCachePriority::move(LRUIterator & it, LRUFileCachePriority & other, const CachePriorityGuard::Lock &) { const auto & entry = *it.getEntry(); if (entry.size == 0) @@ -311,9 +318,8 @@ LRUFileCachePriority::LRUIterator LRUFileCachePriority::move(LRUIterator & it, L if (queue_entry->size != 0 && queue_entry->key == entry.key && queue_entry->offset == entry.offset) throw Exception( ErrorCodes::LOGICAL_ERROR, - "Attempt to add duplicate queue entry to queue. " - "(Key: {}, offset: {}, size: {})", - entry.key, entry.offset, entry.size); + "Attempt to add duplicate queue entry to queue: {}", + entry.toString()); } #endif @@ -395,10 +401,8 @@ void LRUFileCachePriority::LRUIterator::invalidate() assertValid(); const auto & entry = *iterator; - LOG_TEST( - cache_priority->log, - "Invalidating entry in LRU queue. Key: {}, offset: {}, previous size: {}", - entry->key, entry->offset, entry->size); + LOG_TEST(cache_priority->log, + "Invalidating entry in LRU queue entry {}", entry->toString()); chassert(entry->size != 0); cache_priority->updateSize(-entry->size); @@ -406,19 +410,29 @@ void LRUFileCachePriority::LRUIterator::invalidate() entry->size = 0; } -void LRUFileCachePriority::LRUIterator::incrementSize(size_t size, const CachePriorityGuard::Lock &) +void LRUFileCachePriority::LRUIterator::incrementSize(size_t size, const CachePriorityGuard::Lock & lock) { + chassert(size); assertValid(); const auto & entry = *iterator; + + if (!cache_priority->canFit(size, /* elements */0, lock)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot increment size by {} for entry {}. Current state: {}", + size, entry->toString(), cache_priority->getStateInfoForLog(lock)); + } + LOG_TEST( cache_priority->log, - "Increment size with {} in LRU queue for key: {}, offset: {}, previous size: {}", - size, entry->key, entry->offset, entry->size); + "Incrementing size with {} in LRU queue for entry {}", + size, entry->toString()); - chassert(size); cache_priority->updateSize(size); entry->size += size; + + cache_priority->check(lock); } void LRUFileCachePriority::LRUIterator::decrementSize(size_t size) @@ -426,10 +440,9 @@ void LRUFileCachePriority::LRUIterator::decrementSize(size_t size) assertValid(); const auto & entry = *iterator; - LOG_TEST( - cache_priority->log, - "Decrement size with {} in LRU queue for key: {}, offset: {}, previous size: {}", - size, entry->key, entry->offset, entry->size); + LOG_TEST(cache_priority->log, + "Decrement size with {} in LRU queue entry {}", + size, entry->toString()); chassert(size); chassert(entry->size >= size); @@ -438,10 +451,11 @@ void LRUFileCachePriority::LRUIterator::decrementSize(size_t size) entry->size -= size; } -size_t LRUFileCachePriority::LRUIterator::increasePriority(const CachePriorityGuard::Lock &) +size_t LRUFileCachePriority::LRUIterator::increasePriority(const CachePriorityGuard::Lock & lock) { assertValid(); cache_priority->queue.splice(cache_priority->queue.end(), cache_priority->queue, iterator); + cache_priority->check(lock); return ++((*iterator)->hits); } @@ -463,7 +477,17 @@ void LRUFileCachePriority::shuffle(const CachePriorityGuard::Lock &) queue.splice(queue.end(), queue, it); } -void LRUFileCachePriority::holdImpl(size_t size, size_t elements, IteratorPtr, const CachePriorityGuard::Lock & lock) +std::string LRUFileCachePriority::getStateInfoForLog(const CachePriorityGuard::Lock & lock) const +{ + return fmt::format("size: {}/{}, elements: {}/{} (description: {})", + getSize(lock), max_size, getElementsCount(lock), max_elements, description); +} + +void LRUFileCachePriority::holdImpl( + size_t size, + size_t elements, + QueueEntryType /* queue_entry_type */, + const CachePriorityGuard::Lock & lock) { if (!canFit(size, elements, lock)) { @@ -478,7 +502,7 @@ void LRUFileCachePriority::holdImpl(size_t size, size_t elements, IteratorPtr, c state->current_elements_num += elements; } -void LRUFileCachePriority::releaseImpl(size_t size, size_t elements, IteratorPtr) +void LRUFileCachePriority::releaseImpl(size_t size, size_t elements, QueueEntryType /* queue_entry_type */) { state->current_size -= size; state->current_elements_num -= elements; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index d33c63a75c1..1f80cfac170 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -22,7 +22,11 @@ protected: using StatePtr = std::shared_ptr; public: - LRUFileCachePriority(size_t max_size_, size_t max_elements_, StatePtr state_ = nullptr); + LRUFileCachePriority( + size_t max_size_, + size_t max_elements_, + StatePtr state_ = nullptr, + const std::string & description_ = "none"); size_t getSize(const CachePriorityGuard::Lock &) const override { return state->current_size; } @@ -32,6 +36,10 @@ public: size_t getElementsCountApprox() const override { return state->current_elements_num; } + QueueEntryType getDefaultQueueEntryType() const override { return FileCacheQueueEntryType::LRU; } + + std::string getStateInfoForLog(const CachePriorityGuard::Lock & lock) const override; + bool canFit( /// NOLINT size_t size, size_t elements, @@ -77,6 +85,7 @@ private: friend class SLRUFileCachePriority; LRUQueue queue; + const std::string description; LoggerPtr log = getLogger("LRUFileCachePriority"); StatePtr state; @@ -106,8 +115,13 @@ private: LRUIterator move(LRUIterator & it, LRUFileCachePriority & other, const CachePriorityGuard::Lock &); LRUIterator add(EntryPtr entry, const CachePriorityGuard::Lock &); - void holdImpl(size_t size, size_t elements, IteratorPtr, const CachePriorityGuard::Lock & lock) override; - void releaseImpl(size_t size, size_t elements, IteratorPtr) override; + void holdImpl( + size_t size, + size_t elements, + QueueEntryType queue_entry_type, + const CachePriorityGuard::Lock & lock) override; + + void releaseImpl(size_t size, size_t elements, QueueEntryType queue_entry_type) override; }; class LRUFileCachePriority::LRUIterator : public IFileCachePriority::Iterator diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 07dd36e7655..9ea4cffeec0 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -14,7 +14,7 @@ namespace { size_t getRatio(size_t total, double ratio) { - return static_cast(total * std::clamp(ratio, 0.0, 1.0)); + return std::lround(total * std::clamp(ratio, 0.0, 1.0)); } } @@ -26,12 +26,20 @@ SLRUFileCachePriority::SLRUFileCachePriority( LRUFileCachePriority::StatePtr protected_state_) : IFileCachePriority(max_size_, max_elements_) , size_ratio(size_ratio_) - , protected_queue(LRUFileCachePriority(getRatio(max_size_, size_ratio), getRatio(max_elements_, size_ratio), protected_state_)) - , probationary_queue(LRUFileCachePriority(getRatio(max_size_, 1 - size_ratio), getRatio(max_elements_, 1 - size_ratio), probationary_state_)) + , protected_queue(LRUFileCachePriority(getRatio(max_size_, size_ratio), + getRatio(max_elements_, size_ratio), + protected_state_, + "protected")) + , probationary_queue(LRUFileCachePriority(getRatio(max_size_, 1 - size_ratio), + getRatio(max_elements_, 1 - size_ratio), + probationary_state_, + "probationary")) { LOG_DEBUG( - log, "Using probationary queue size: {}, protected queue size: {}", - probationary_queue.max_size, protected_queue.max_elements); + log, "Probationary queue {} in size and {} in elements. " + "Protected queue {} in size and {} in elements", + probationary_queue.max_size, probationary_queue.max_elements, + protected_queue.max_size, protected_queue.max_elements); } size_t SLRUFileCachePriority::getSize(const CachePriorityGuard::Lock & lock) const @@ -84,6 +92,7 @@ IFileCachePriority::IteratorPtr SLRUFileCachePriority::add( /// NOLINT const CachePriorityGuard::Lock & lock, bool is_startup) { + IteratorPtr iterator; if (is_startup) { /// If it is server startup, we put entries in any queue it will fit in, @@ -92,19 +101,28 @@ IFileCachePriority::IteratorPtr SLRUFileCachePriority::add( /// NOLINT if (probationary_queue.canFit(size, 1, lock)) { auto lru_iterator = probationary_queue.add(std::make_shared(key_metadata->key, offset, size, key_metadata), lock); - return std::make_shared(this, std::move(lru_iterator), false); + iterator = std::make_shared(this, std::move(lru_iterator), false); } else { auto lru_iterator = protected_queue.add(std::make_shared(key_metadata->key, offset, size, key_metadata), lock); - return std::make_shared(this, std::move(lru_iterator), true); + iterator = std::make_shared(this, std::move(lru_iterator), true); } } else { auto lru_iterator = probationary_queue.add(std::make_shared(key_metadata->key, offset, size, key_metadata), lock); - return std::make_shared(this, std::move(lru_iterator), false); + iterator = std::make_shared(this, std::move(lru_iterator), false); } + + if (getSize(lock) > max_size || getElementsCount(lock) > max_elements) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Went beyond limits. Added {} for {} element ({}:{}). Current state: {}", + size, iterator->getType(), key_metadata->key, offset, getStateInfoForLog(lock)); + } + + return iterator; } bool SLRUFileCachePriority::collectCandidatesForEviction( @@ -137,45 +155,116 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( /// Entry is in protected queue. /// Check if we have enough space in protected queue to fit a new size of entry. /// `size` is the increment to the current entry.size we want to increase. - if (protected_queue.canFit(size, 1, lock)) + /// Here `elements` is 0, because entry is already in the protected queue. + if (protected_queue.canFit(size, /* elements */0, lock)) + { return true; + } /// If not enough space - we need to "downgrade" lowest priority entries from protected /// queue to probationary queue. /// The amount of such "downgraded" entries is equal to the amount /// required to make space for additionary `size` bytes for entry. auto downgrade_candidates = std::make_shared(); - FileCacheReserveStat downgrade_stat; if (!protected_queue.collectCandidatesForEviction( - size, downgrade_stat, *downgrade_candidates, reservee, user_id, reached_size_limit, reached_elements_limit, lock)) - return false; - - const size_t size_to_downgrade = downgrade_stat.stat.releasable_size; - - bool reached_size_limit_noop; - bool reached_elements_limit_noop; - if (!probationary_queue.canFit(size_to_downgrade, 1, lock) - && !probationary_queue.collectCandidatesForEviction( - size_to_downgrade, stat, res, reservee, user_id, reached_size_limit_noop, reached_elements_limit_noop, lock)) - return false; - - res.setFinalizeEvictionFunc([=, this](const CachePriorityGuard::Lock & lk) mutable + size, stat, *downgrade_candidates, reservee, + user_id, reached_size_limit, reached_elements_limit, lock)) { + return false; + } + + const size_t size_to_downgrade = stat.stat.releasable_size; + const size_t elements_to_downgrade = stat.stat.releasable_count; + + chassert(size_to_downgrade); + + FileCacheReserveStat probationary_stat; + bool downgrade_reached_size_limit = false; + bool downgrade_reached_elements_limit = false; + if (!probationary_queue.canFit(size_to_downgrade, elements_to_downgrade, lock) + && !probationary_queue.collectCandidatesForEviction( + size_to_downgrade, probationary_stat, res, reservee, user_id, + downgrade_reached_size_limit, downgrade_reached_elements_limit, lock)) + { + return false; + } + + if (downgrade_candidates->size() == 0) + { + return true; + } + + const bool downgrade_after_eviction = res.size() > 0; + auto take_space_hold = [&]() + { + const size_t hold_size = downgrade_reached_size_limit + ? size_to_downgrade > probationary_stat.stat.releasable_size ? size - probationary_stat.stat.releasable_size : 0 + : size_to_downgrade; + + const size_t hold_elements = downgrade_reached_elements_limit + ? elements_to_downgrade > probationary_stat.stat.releasable_count ? size - probationary_stat.stat.releasable_count : 0 + : elements_to_downgrade; + + return std::make_shared( + hold_size, hold_elements, QueueEntryType::SLRU_Probationary, probationary_queue, lock); + }; + + auto downgrade_func = [=, holder = downgrade_after_eviction ? take_space_hold() : nullptr, this] + (const CachePriorityGuard::Lock & lk) + { + if (holder) + holder->release(); + + LOG_TEST(log, "Downgrading {} elements from protected to probationary. " + "Total size: {}", + downgrade_candidates->size(), stat.stat.releasable_size); + for (const auto & [key, key_candidates] : *downgrade_candidates) { for (const auto & candidate : key_candidates.candidates) - { - auto * candidate_it = assert_cast(candidate->getQueueIterator().get()); - candidate_it->lru_iterator = probationary_queue.move(candidate_it->lru_iterator, protected_queue, lk); - candidate_it->is_protected = false; - } + downgrade(candidate->getQueueIterator(), lk); } - }); + }; + + if (downgrade_after_eviction) + { + /// Downgrade from protected to probationary only after + /// we free up space in probationary (in order to fit these downgrade candidates). + res.setFinalizeEvictionFunc(std::move(downgrade_func)); + } + else + { + /// Enough space in probationary queue already to fit our downgrade candidates. + downgrade_func(lock); + } return true; } +void SLRUFileCachePriority::downgrade(IteratorPtr iterator, const CachePriorityGuard::Lock & lock) +{ + auto * candidate_it = assert_cast(iterator.get()); + if (!candidate_it->is_protected) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot downgrade {}: it is already in probationary queue", + candidate_it->getEntry()->toString()); + } + + const size_t entry_size = candidate_it->entry->size; + if (!probationary_queue.canFit(entry_size, 1, lock)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot downgrade {}: not enough space: {}", + candidate_it->getEntry()->toString(), + probationary_queue.getStateInfoForLog(lock)); + } + + candidate_it->lru_iterator = probationary_queue.move(candidate_it->lru_iterator, protected_queue, lock); + candidate_it->is_protected = false; +} + void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const CachePriorityGuard::Lock & lock) { /// If entry is already in protected queue, @@ -188,8 +277,8 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach /// Entry is in probationary queue. /// We need to move it to protected queue. - const size_t size = iterator.getEntry()->size; - if (size > protected_queue.getSizeLimit(lock)) + const size_t entry_size = iterator.getEntry()->size; + if (entry_size > protected_queue.getSizeLimit(lock)) { /// Entry size is bigger than the whole protected queue limit. /// This is only possible if protected_queue_size_limit is less than max_file_segment_size, @@ -208,7 +297,8 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach bool reached_elements_limit_noop; if (!protected_queue.collectCandidatesForEviction( - size, downgrade_stat, downgrade_candidates, {}, "", reached_size_limit_noop, reached_elements_limit_noop, lock)) + entry_size, downgrade_stat, downgrade_candidates, {}, "", + reached_size_limit_noop, reached_elements_limit_noop, lock)) { /// We cannot make space for entry to be moved to protected queue /// (not enough releasable file segments). @@ -217,12 +307,17 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach return; } - /// The amount of such "downgraded" entries is equal to the amount - /// required to make space for entry we want to insert. - const size_t size_to_downgrade = downgrade_stat.stat.releasable_count; + const size_t downgrade_size = downgrade_stat.stat.releasable_size; + const size_t downgrade_count = downgrade_stat.stat.releasable_count; + + /// Then we need to free up space in probationary for downgrade candidates, + /// but we take into account that we'll remove reservee from probationary + /// at the same time, so recalculate size which we need to free. size_t size_to_free = 0; - if (size_to_downgrade && size_to_downgrade > size) - size_to_free = size_to_downgrade - size; + if (downgrade_size && downgrade_size > entry_size) + size_to_free = downgrade_size - entry_size; + + LOG_TEST(log, "Will free up {} from probationary", size_to_free); /// Now we need to check if those "downgrade" candidates can actually /// be moved to probationary queue. @@ -232,7 +327,8 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach if (size_to_free) { if (!probationary_queue.collectCandidatesForEviction( - size_to_free, stat, eviction_candidates, {}, {}, reached_size_limit_noop, reached_elements_limit_noop, lock)) + size_to_free, stat, eviction_candidates, {}, {}, + reached_size_limit_noop, reached_elements_limit_noop, lock)) { /// "downgrade" candidates cannot be moved to probationary queue, /// so entry cannot be moved to protected queue as well. @@ -247,17 +343,29 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach /// All checks passed, now we can move downgrade candidates to /// probationary queue and our entry to protected queue. + EntryPtr entry = iterator.getEntry(); + /// We need to remove the entry from probationary first + /// in order to make space for downgrade from protected. iterator.lru_iterator.remove(lock); + if (!probationary_queue.canFit(downgrade_size, downgrade_count, lock)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot downgrade {} elements by total size of {}: not enough space: {}", + downgrade_size, downgrade_count, + probationary_queue.getStateInfoForLog(lock)); + } + for (const auto & [key, key_candidates] : downgrade_candidates) { + LOG_TEST(log, "Downgrading {} elements from protected to probationary. " + "Total size: {}, current probationary state: {}", + downgrade_candidates.size(), downgrade_size, + probationary_queue.getStateInfoForLog(lock)); + for (const auto & candidate : key_candidates.candidates) - { - auto * candidate_it = assert_cast(candidate->getQueueIterator().get()); - candidate_it->lru_iterator = probationary_queue.move(candidate_it->lru_iterator, protected_queue, lock); - candidate_it->is_protected = false; - } + downgrade(candidate->getQueueIterator(), lock); } iterator.lru_iterator = protected_queue.add(entry, lock); @@ -313,6 +421,7 @@ size_t SLRUFileCachePriority::SLRUIterator::increasePriority(const CachePriority { assertValid(); cache_priority->increasePriority(*this, lock); + cache_priority->check(lock); return getEntry()->hits; } @@ -345,31 +454,67 @@ void SLRUFileCachePriority::SLRUIterator::assertValid() const lru_iterator.assertValid(); } -void SLRUFileCachePriority::holdImpl(size_t size, size_t elements, IteratorPtr reservee, const CachePriorityGuard::Lock & lock) +void SLRUFileCachePriority::holdImpl( + size_t size, + size_t elements, + QueueEntryType queue_entry_type, + const CachePriorityGuard::Lock & lock) { - if (reservee) + switch (queue_entry_type) { - const auto * slru_iterator = assert_cast(reservee.get()); - if (slru_iterator->is_protected) - return protected_queue.holdImpl(size, elements, reservee, lock); - else - return probationary_queue.holdImpl(size, elements, reservee, lock); + case QueueEntryType::SLRU_Protected: + { + protected_queue.holdImpl(size, elements, queue_entry_type, lock); + break; + } + case QueueEntryType::SLRU_Probationary: + { + probationary_queue.holdImpl(size, elements, queue_entry_type, lock); + break; + } + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Unexpected queue entry type: {}", queue_entry_type); } - else - return probationary_queue.holdImpl(size, elements, reservee, lock); } -void SLRUFileCachePriority::releaseImpl(size_t size, size_t elements, IteratorPtr reservee) +void SLRUFileCachePriority::releaseImpl(size_t size, size_t elements, QueueEntryType queue_entry_type) { - if (reservee) + switch (queue_entry_type) { - const auto * slru_iterator = assert_cast(reservee.get()); - if (slru_iterator->is_protected) - return protected_queue.releaseImpl(size, elements, reservee); - else - return probationary_queue.releaseImpl(size, elements, reservee); + case QueueEntryType::SLRU_Protected: + { + protected_queue.releaseImpl(size, elements, queue_entry_type); + break; + } + case QueueEntryType::SLRU_Probationary: + { + probationary_queue.releaseImpl(size, elements, queue_entry_type); + break; + } + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Unexpected queue entry type: {}", queue_entry_type); } - else - return probationary_queue.releaseImpl(size, elements, reservee); } + +std::string SLRUFileCachePriority::getStateInfoForLog(const CachePriorityGuard::Lock & lock) const +{ + return fmt::format("total size {}/{}, elements {}/{}, " + "probationary queue size {}/{}, elements {}/{}, " + "protected queue size {}/{}, elements {}/{}", + getSize(lock), max_size, getElementsCount(lock), max_elements, + probationary_queue.getSize(lock), probationary_queue.max_size, + probationary_queue.getElementsCount(lock), probationary_queue.max_elements, + protected_queue.getSize(lock), protected_queue.max_size, + protected_queue.getElementsCount(lock), protected_queue.max_elements); +} + +void SLRUFileCachePriority::check(const CachePriorityGuard::Lock & lock) const +{ + probationary_queue.check(lock); + protected_queue.check(lock); + IFileCachePriority::check(lock); +} + } diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h index e8c7bbc5b0c..25ee967b6f1 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.h +++ b/src/Interpreters/Cache/SLRUFileCachePriority.h @@ -29,6 +29,12 @@ public: size_t getElementsCountApprox() const override; + QueueEntryType getDefaultQueueEntryType() const override { return FileCacheQueueEntryType::SLRU_Probationary; } + + std::string getStateInfoForLog(const CachePriorityGuard::Lock & lock) const override; + + void check(const CachePriorityGuard::Lock &) const override; + bool canFit( /// NOLINT size_t size, size_t elements, @@ -68,8 +74,15 @@ private: void increasePriority(SLRUIterator & iterator, const CachePriorityGuard::Lock & lock); - void holdImpl(size_t size, size_t elements, IteratorPtr reservee, const CachePriorityGuard::Lock & lock) override; - void releaseImpl(size_t size, size_t elements, IteratorPtr reservee) override; + void holdImpl( + size_t size, + size_t elements, + QueueEntryType queue_entry_type, + const CachePriorityGuard::Lock & lock) override; + + void releaseImpl(size_t size, size_t elements, QueueEntryType queue_entry_type) override; + + void downgrade(IteratorPtr iterator, const CachePriorityGuard::Lock &); }; class SLRUFileCachePriority::SLRUIterator : public IFileCachePriority::Iterator From 6f4ce33f027be0600eea4e563ff06a5eac0894f5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 20 Mar 2024 22:12:57 +0100 Subject: [PATCH 0312/1165] Change how fuzzer arguments are parsed --- programs/local/LocalServer.cpp | 12 ------------ src/Client/ClientBase.cpp | 19 +++++++++++++++++-- src/Client/ClientBase.h | 4 +--- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index f485e3880c1..6f7d3681c46 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -878,11 +878,6 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector &, std::vector &) { -#if defined(FUZZING_MODE) - if (argc) - fuzzer_args.push_back(argv[0]); -#endif - for (int arg_num = 1; arg_num < argc; ++arg_num) { std::string_view arg = argv[arg_num]; @@ -918,13 +913,6 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum arg = argv[arg_num]; addMultiquery(arg, common_arguments); } - else if (arg == "--") - { -#if defined(FUZZING_MODE) - fuzzer_args.insert(fuzzer_args.end(), &argv[arg_num + 1], &argv[argc]); - break; -#endif - } else { common_arguments.emplace_back(arg); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index c4cb6e1a5ad..05f583ef438 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -16,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -70,6 +68,7 @@ #include #include +#include #include #include #include @@ -2638,6 +2637,16 @@ ClientBase * app; void ClientBase::runLibFuzzer() { app = this; + std::vector fuzzer_args_holder; + + if (const char * fuzzer_args_env = getenv("FUZZER_ARGS")) // NOLINT(concurrency-mt-unsafe) + boost::split(fuzzer_args_holder, fuzzer_args_env, isWhitespaceASCII, boost::token_compress_on); + + std::vector fuzzer_args; + fuzzer_args.push_back(argv0); + for (auto & arg : fuzzer_args_holder) + fuzzer_args.emplace_back(arg.data()); + int fuzzer_argc = fuzzer_args.size(); char ** fuzzer_argv = fuzzer_args.data(); @@ -2656,6 +2665,8 @@ void ClientBase::runLibFuzzer() return 0; }); } +#else +void ClientBase::runLibFuzzer() {} #endif @@ -2828,6 +2839,8 @@ void ClientBase::init(int argc, char ** argv) Arguments common_arguments = {""}; /// 0th argument is ignored. std::vector hosts_and_ports_arguments; + if (argc) + argv0 = argv[0]; readArguments(argc, argv, common_arguments, external_tables_arguments, hosts_and_ports_arguments); /// Support for Unicode dashes @@ -2892,6 +2905,8 @@ void ClientBase::init(int argc, char ** argv) ("interactive", "Process queries-file or --query query and start interactive mode") ("pager", po::value(), "Pipe all output into this command (less or similar)") ("max_memory_usage_in_client", po::value(), "Set memory limit in client/local server") + + ("fuzzer-args", po::value(), "Command line arguments for the LLVM's libFuzzer driver. Only relevant if the application is compiled with libFuzzer.") ; addOptions(options_description); diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index c0052ade7aa..51dfaebd995 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -78,10 +78,8 @@ protected: void runInteractive(); void runNonInteractive(); -#if defined(FUZZING_MODE) - std::vector fuzzer_args; + char * argv0 = nullptr; void runLibFuzzer(); -#endif virtual bool processWithFuzzing(const String &) { From a31cf43499b1d02827ee3ca08e3fc961c3911eaa Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 20 Mar 2024 23:13:17 +0100 Subject: [PATCH 0313/1165] Update the dictionary for OSSFuzz (#61672) --- tests/fuzz/README.md | 18 +- tests/fuzz/all.dict | 3455 +++++++++++++++- tests/fuzz/dictionaries/datatypes.dict | 287 +- tests/fuzz/dictionaries/functions.dict | 5264 +++++++++++++++++++----- tests/fuzz/dictionaries/key_words.dict | 426 +- 5 files changed, 8089 insertions(+), 1361 deletions(-) diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 5b864babde3..6b5b161b2d5 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -1,13 +1,23 @@ -The list of functions generated via following query +The list of functions generated via the following query ``` - clickhouse-client -q "select concat('\"', name, '\"') from system.functions union all select concat('\"', alias_to, '\"') from system.functions where alias_to != '' " > functions.dict + clickhouse client -q "SELECT * FROM (SELECT DISTINCT concat('\"', name, '\"') as res FROM system.functions ORDER BY name UNION ALL SELECT concat('\"', a.name, b.name, '\"') as res FROM system.functions as a CROSS JOIN system.aggregate_function_combinators as b WHERE a.is_aggregate = 1) ORDER BY res" > functions.dict ``` -The list of datatypes generated via following query: +The list of datatypes generated via the following query: ``` - clickhouse-client -q "select concat('\"', name, '\"') from system.data_type_families union all select concat('\"', alias_to, '\"') from system.data_type_families where alias_to != '' " > datatypes.dict + clickhouse client -q "SELECT DISTINCT concat('\"', name, '\"') as res FROM system.data_type_families ORDER BY name" > datatypes.dict +``` + +The list of keywords generated via the following query: + +``` + clickhouse client -q "SELECT DISTINCT concat('\"', keyword, '\"') as res FROM system.keywords ORDER BY keyword" > key_words.dict ``` Then merge all dictionaries into one (all.dict) + +``` + cat ./dictionaries/* | sort | uniq > all.dict +``` \ No newline at end of file diff --git a/tests/fuzz/all.dict b/tests/fuzz/all.dict index b38e2c11a1c..f08e319f0d4 100644 --- a/tests/fuzz/all.dict +++ b/tests/fuzz/all.dict @@ -1,59 +1,333 @@ - "abs" -"accurate_Cast" "accurateCast" -"accurate_CastOrNull" +"accurateCastOrDefault" "accurateCastOrNull" "acos" "acosh" "ADD" "ADD COLUMN" "ADD CONSTRAINT" +"addDate" "addDays" "addHours" "ADD INDEX" +"addInterval" +"addMicroseconds" +"addMilliseconds" "addMinutes" "addMonths" +"addNanoseconds" +"ADD PROJECTION" "addQuarters" "addressToLine" +"addressToLineWithInlines" "addressToSymbol" "addSeconds" +"ADD STATISTIC" +"addTupleOfIntervals" "addWeeks" "addYears" +"ADMIN OPTION FOR" "aes_decrypt_mysql" "aes_encrypt_mysql" "AFTER" +"age" "AggregateFunction" "aggThrow" +"aggThrowArgMax" +"aggThrowArgMin" +"aggThrowArray" +"aggThrowDistinct" +"aggThrowForEach" +"aggThrowIf" +"aggThrowMap" +"aggThrowMerge" +"aggThrowNull" +"aggThrowOrDefault" +"aggThrowOrNull" +"aggThrowResample" +"aggThrowSimpleState" +"aggThrowState" +"ALGORITHM" "ALIAS" "ALL" +"ALLOWED_LATENESS" "alphaTokens" "ALTER" +"ALTER COLUMN" +"ALTER DATABASE" "ALTER LIVE VIEW" +"ALTER POLICY" +"ALTER PROFILE" +"ALTER QUOTA" +"ALTER ROLE" +"ALTER ROW POLICY" +"ALTER SETTINGS PROFILE" "ALTER TABLE" +"ALTER TEMPORARY TABLE" +"ALTER USER" +"analysisOfVariance" +"analysisOfVarianceArgMax" +"analysisOfVarianceArgMin" +"analysisOfVarianceArray" +"analysisOfVarianceDistinct" +"analysisOfVarianceForEach" +"analysisOfVarianceIf" +"analysisOfVarianceMap" +"analysisOfVarianceMerge" +"analysisOfVarianceNull" +"analysisOfVarianceOrDefault" +"analysisOfVarianceOrNull" +"analysisOfVarianceResample" +"analysisOfVarianceSimpleState" +"analysisOfVarianceState" "and" "AND" +"AND STDOUT" +"anova" +"anovaArgMax" +"anovaArgMin" +"anovaArray" +"anovaDistinct" +"anovaForEach" +"anovaIf" +"anovaMap" +"anovaMerge" +"anovaNull" +"anovaOrDefault" +"anovaOrNull" +"anovaResample" +"anovaSimpleState" +"anovaState" "ANTI" "any" "ANY" +"anyArgMax" +"anyArgMin" +"anyArray" +"anyDistinct" +"anyForEach" "anyHeavy" +"anyHeavyArgMax" +"anyHeavyArgMin" +"anyHeavyArray" +"anyHeavyDistinct" +"anyHeavyForEach" +"anyHeavyIf" +"anyHeavyMap" +"anyHeavyMerge" +"anyHeavyNull" +"anyHeavyOrDefault" +"anyHeavyOrNull" +"anyHeavyResample" +"anyHeavySimpleState" +"anyHeavyState" +"anyIf" "anyLast" +"anyLastArgMax" +"anyLastArgMin" +"anyLastArray" +"anyLastDistinct" +"anyLastForEach" +"anyLastIf" +"anyLastMap" +"anyLastMerge" +"anyLastNull" +"anyLastOrDefault" +"anyLastOrNull" +"anyLastResample" +"anyLast_respect_nulls" +"anyLast_respect_nullsArgMax" +"anyLast_respect_nullsArgMin" +"anyLast_respect_nullsArray" +"anyLast_respect_nullsDistinct" +"anyLast_respect_nullsForEach" +"anyLast_respect_nullsIf" +"anyLast_respect_nullsMap" +"anyLast_respect_nullsMerge" +"anyLast_respect_nullsNull" +"anyLast_respect_nullsOrDefault" +"anyLast_respect_nullsOrNull" +"anyLast_respect_nullsResample" +"anyLast_respect_nullsSimpleState" +"anyLast_respect_nullsState" +"anyLastSimpleState" +"anyLastState" +"anyMap" +"anyMerge" +"anyNull" +"anyOrDefault" +"anyOrNull" +"anyResample" +"any_respect_nulls" +"any_respect_nullsArgMax" +"any_respect_nullsArgMin" +"any_respect_nullsArray" +"any_respect_nullsDistinct" +"any_respect_nullsForEach" +"any_respect_nullsIf" +"any_respect_nullsMap" +"any_respect_nullsMerge" +"any_respect_nullsNull" +"any_respect_nullsOrDefault" +"any_respect_nullsOrNull" +"any_respect_nullsResample" +"any_respect_nullsSimpleState" +"any_respect_nullsState" +"anySimpleState" +"anyState" +"any_value" +"any_valueArgMax" +"any_valueArgMin" +"any_valueArray" +"any_valueDistinct" +"any_valueForEach" +"any_valueIf" +"any_valueMap" +"any_valueMerge" +"any_valueNull" +"any_valueOrDefault" +"any_valueOrNull" +"any_valueResample" +"any_value_respect_nulls" +"any_value_respect_nullsArgMax" +"any_value_respect_nullsArgMin" +"any_value_respect_nullsArray" +"any_value_respect_nullsDistinct" +"any_value_respect_nullsForEach" +"any_value_respect_nullsIf" +"any_value_respect_nullsMap" +"any_value_respect_nullsMerge" +"any_value_respect_nullsNull" +"any_value_respect_nullsOrDefault" +"any_value_respect_nullsOrNull" +"any_value_respect_nullsResample" +"any_value_respect_nullsSimpleState" +"any_value_respect_nullsState" +"any_valueSimpleState" +"any_valueState" +"APPEND" "appendTrailingCharIfAbsent" +"APPLY" +"APPLY DELETED MASK" +"approx_top_count" +"approx_top_countArgMax" +"approx_top_countArgMin" +"approx_top_countArray" +"approx_top_countDistinct" +"approx_top_countForEach" +"approx_top_countIf" +"approx_top_countMap" +"approx_top_countMerge" +"approx_top_countNull" +"approx_top_countOrDefault" +"approx_top_countOrNull" +"approx_top_countResample" +"approx_top_countSimpleState" +"approx_top_countState" +"approx_top_k" +"approx_top_kArgMax" +"approx_top_kArgMin" +"approx_top_kArray" +"approx_top_kDistinct" +"approx_top_kForEach" +"approx_top_kIf" +"approx_top_kMap" +"approx_top_kMerge" +"approx_top_kNull" +"approx_top_kOrDefault" +"approx_top_kOrNull" +"approx_top_kResample" +"approx_top_kSimpleState" +"approx_top_kState" +"approx_top_sum" +"approx_top_sumArgMax" +"approx_top_sumArgMin" +"approx_top_sumArray" +"approx_top_sumDistinct" +"approx_top_sumForEach" +"approx_top_sumIf" +"approx_top_sumMap" +"approx_top_sumMerge" +"approx_top_sumNull" +"approx_top_sumOrDefault" +"approx_top_sumOrNull" +"approx_top_sumResample" +"approx_top_sumSimpleState" +"approx_top_sumState" "argMax" +"argMaxArgMax" +"argMaxArgMin" +"argMaxArray" +"argMaxDistinct" +"argMaxForEach" +"argMaxIf" +"argMaxMap" +"argMaxMerge" +"argMaxNull" +"argMaxOrDefault" +"argMaxOrNull" +"argMaxResample" +"argMaxSimpleState" +"argMaxState" "argMin" +"argMinArgMax" +"argMinArgMin" +"argMinArray" +"argMinDistinct" +"argMinForEach" +"argMinIf" +"argMinMap" +"argMinMerge" +"argMinNull" +"argMinOrDefault" +"argMinOrNull" +"argMinResample" +"argMinSimpleState" +"argMinState" "array" "Array" -"ARRAY" +"array_agg" +"array_aggArgMax" +"array_aggArgMin" +"array_aggArray" +"array_aggDistinct" +"array_aggForEach" +"array_aggIf" +"array_aggMap" +"array_aggMerge" +"array_aggNull" +"array_aggOrDefault" +"array_aggOrNull" +"array_aggResample" +"array_aggSimpleState" +"array_aggState" "arrayAll" "arrayAUC" "arrayAvg" "arrayCompact" "arrayConcat" +"array_concat_agg" +"array_concat_aggArgMax" +"array_concat_aggArgMin" +"array_concat_aggArray" +"array_concat_aggDistinct" +"array_concat_aggForEach" +"array_concat_aggIf" +"array_concat_aggMap" +"array_concat_aggMerge" +"array_concat_aggNull" +"array_concat_aggOrDefault" +"array_concat_aggOrNull" +"array_concat_aggResample" +"array_concat_aggSimpleState" +"array_concat_aggState" "arrayCount" "arrayCumSum" "arrayCumSumNonNegative" "arrayDifference" "arrayDistinct" +"arrayDotProduct" "arrayElement" "arrayEnumerate" "arrayEnumerateDense" @@ -65,10 +339,16 @@ "arrayFilter" "arrayFirst" "arrayFirstIndex" +"arrayFirstOrNull" "arrayFlatten" +"arrayFold" "arrayIntersect" +"arrayJaccardIndex" "arrayJoin" "ARRAY JOIN" +"arrayLast" +"arrayLastIndex" +"arrayLastOrNull" "arrayMap" "arrayMax" "arrayMin" @@ -80,6 +360,7 @@ "arrayProduct" "arrayPushBack" "arrayPushFront" +"arrayRandomSample" "arrayReduce" "arrayReduceInRanges" "arrayResize" @@ -87,6 +368,11 @@ "arrayReverseFill" "arrayReverseSort" "arrayReverseSplit" +"arrayRotateLeft" +"arrayRotateRight" +"arrayShiftLeft" +"arrayShiftRight" +"arrayShingles" "arrayShuffle" "arraySlice" "arraySort" @@ -99,9 +385,11 @@ "AS" "ASC" "ASCENDING" +"ascii" "asin" "asinh" "ASOF" +"ASSUME" "assumeNotNull" "AST" "ASYNC" @@ -111,14 +399,59 @@ "ATTACH" "ATTACH PART" "ATTACH PARTITION" +"ATTACH POLICY" +"ATTACH PROFILE" +"ATTACH QUOTA" +"ATTACH ROLE" +"ATTACH ROW POLICY" +"ATTACH SETTINGS PROFILE" +"ATTACH USER" +"AUTO_INCREMENT" "avg" +"avgArgMax" +"avgArgMin" +"avgArray" +"avgDistinct" +"avgForEach" +"avgIf" +"avgMap" +"avgMerge" +"avgNull" +"avgOrDefault" +"avgOrNull" +"avgResample" +"avgSimpleState" +"avgState" "avgWeighted" +"avgWeightedArgMax" +"avgWeightedArgMin" +"avgWeightedArray" +"avgWeightedDistinct" +"avgWeightedForEach" +"avgWeightedIf" +"avgWeightedMap" +"avgWeightedMerge" +"avgWeightedNull" +"avgWeightedOrDefault" +"avgWeightedOrNull" +"avgWeightedResample" +"avgWeightedSimpleState" +"avgWeightedState" +"AZURE" +"BACKUP" +"bagexpansion" "bar" +"base58Decode" +"base58Encode" "base64Decode" "base64Encode" +"base_backup" "basename" -"bayesAB" +"BCRYPT_HASH" +"BCRYPT_PASSWORD" +"BEGIN TRANSACTION" "BETWEEN" +"BIDIRECTIONAL" "BIGINT" "BIGINT SIGNED" "BIGINT UNSIGNED" @@ -126,8 +459,23 @@ "BINARY" "BINARY LARGE OBJECT" "BINARY VARYING" +"BIT" "bitAnd" "BIT_AND" +"BIT_ANDArgMax" +"BIT_ANDArgMin" +"BIT_ANDArray" +"BIT_ANDDistinct" +"BIT_ANDForEach" +"BIT_ANDIf" +"BIT_ANDMap" +"BIT_ANDMerge" +"BIT_ANDNull" +"BIT_ANDOrDefault" +"BIT_ANDOrNull" +"BIT_ANDResample" +"BIT_ANDSimpleState" +"BIT_ANDState" "__bitBoolMaskAnd" "__bitBoolMaskOr" "bitCount" @@ -156,11 +504,26 @@ "bitNot" "bitOr" "BIT_OR" +"BIT_ORArgMax" +"BIT_ORArgMin" +"BIT_ORArray" +"BIT_ORDistinct" +"BIT_ORForEach" +"BIT_ORIf" +"BIT_ORMap" +"BIT_ORMerge" +"BIT_ORNull" +"BIT_OROrDefault" +"BIT_OROrNull" +"BIT_ORResample" +"BIT_ORSimpleState" +"BIT_ORState" "bitPositionsToArray" "bitRotateLeft" "bitRotateRight" "bitShiftLeft" "bitShiftRight" +"bitSlice" "__bitSwapLastTwo" "bitTest" "bitTestAll" @@ -168,19 +531,53 @@ "__bitWrapperFunc" "bitXor" "BIT_XOR" +"BIT_XORArgMax" +"BIT_XORArgMin" +"BIT_XORArray" +"BIT_XORDistinct" +"BIT_XORForEach" +"BIT_XORIf" +"BIT_XORMap" +"BIT_XORMerge" +"BIT_XORNull" +"BIT_XOROrDefault" +"BIT_XOROrNull" +"BIT_XORResample" +"BIT_XORSimpleState" +"BIT_XORState" +"BLAKE3" "BLOB" "blockNumber" "blockSerializedSize" "blockSize" -"BOOL" -"BOOLEAN" +"bool" +"Bool" +"boolean" "BOTH" "boundingRatio" +"boundingRatioArgMax" +"boundingRatioArgMin" +"boundingRatioArray" +"boundingRatioDistinct" +"boundingRatioForEach" +"boundingRatioIf" +"boundingRatioMap" +"boundingRatioMerge" +"boundingRatioNull" +"boundingRatioOrDefault" +"boundingRatioOrNull" +"boundingRatioResample" +"boundingRatioSimpleState" +"boundingRatioState" "buildId" "BY" "BYTE" "BYTEA" +"byteHammingDistance" "byteSize" +"byteSlice" +"byteSwap" +"CASCADE" "CASE" "caseWithExpr" "caseWithExpression" @@ -188,10 +585,28 @@ "caseWithoutExpression" "_CAST" "CAST" +"catboostEvaluate" "categoricalInformationValue" +"categoricalInformationValueArgMax" +"categoricalInformationValueArgMin" +"categoricalInformationValueArray" +"categoricalInformationValueDistinct" +"categoricalInformationValueForEach" +"categoricalInformationValueIf" +"categoricalInformationValueMap" +"categoricalInformationValueMerge" +"categoricalInformationValueNull" +"categoricalInformationValueOrDefault" +"categoricalInformationValueOrNull" +"categoricalInformationValueResample" +"categoricalInformationValueSimpleState" +"categoricalInformationValueState" "cbrt" "ceil" "ceiling" +"CHANGE" +"CHANGEABLE_IN_READONLY" +"CHANGED" "char" "CHAR" "CHARACTER" @@ -202,13 +617,19 @@ "CHAR_LENGTH" "CHAR VARYING" "CHECK" +"CHECK ALL TABLES" "CHECK TABLE" "cityHash64" -"CLEAR" +"CLEANUP" "CLEAR COLUMN" "CLEAR INDEX" +"CLEAR PROJECTION" +"CLEAR STATISTIC" "CLOB" "CLUSTER" +"cluster_host_ids" +"CLUSTERS" +"CN" "coalesce" "CODEC" "COLLATE" @@ -216,52 +637,301 @@ "COLUMNS" "COMMENT" "COMMENT COLUMN" +"COMMIT" +"COMPRESSION" "concat" "concatAssumeInjective" +"concatWithSeparator" +"concatWithSeparatorAssumeInjective" +"concat_ws" "connection_id" -"connectionid" "connectionId" +"CONST" "CONSTRAINT" +"contingency" +"contingencyArgMax" +"contingencyArgMin" +"contingencyArray" +"contingencyDistinct" +"contingencyForEach" +"contingencyIf" +"contingencyMap" +"contingencyMerge" +"contingencyNull" +"contingencyOrDefault" +"contingencyOrNull" +"contingencyResample" +"contingencySimpleState" +"contingencyState" "convertCharset" "corr" +"corrArgMax" +"corrArgMin" +"corrArray" +"corrDistinct" +"corrForEach" +"corrIf" +"corrMap" +"corrMatrix" +"corrMatrixArgMax" +"corrMatrixArgMin" +"corrMatrixArray" +"corrMatrixDistinct" +"corrMatrixForEach" +"corrMatrixIf" +"corrMatrixMap" +"corrMatrixMerge" +"corrMatrixNull" +"corrMatrixOrDefault" +"corrMatrixOrNull" +"corrMatrixResample" +"corrMatrixSimpleState" +"corrMatrixState" +"corrMerge" +"corrNull" +"corrOrDefault" +"corrOrNull" +"corrResample" +"corrSimpleState" "corrStable" +"corrStableArgMax" +"corrStableArgMin" +"corrStableArray" +"corrStableDistinct" +"corrStableForEach" +"corrStableIf" +"corrStableMap" +"corrStableMerge" +"corrStableNull" +"corrStableOrDefault" +"corrStableOrNull" +"corrStableResample" +"corrStableSimpleState" +"corrStableState" +"corrState" "cos" "cosh" +"cosineDistance" "count" +"countArgMax" +"countArgMin" +"countArray" "countDigits" +"countDistinct" "countEqual" +"countForEach" +"countIf" +"countMap" "countMatches" "countMatchesCaseInsensitive" +"countMerge" +"countNull" +"countOrDefault" +"countOrNull" +"countResample" +"countSimpleState" +"countState" "countSubstrings" "countSubstringsCaseInsensitive" "countSubstringsCaseInsensitiveUTF8" "covarPop" "COVAR_POP" +"covarPopArgMax" +"COVAR_POPArgMax" +"covarPopArgMin" +"COVAR_POPArgMin" +"covarPopArray" +"COVAR_POPArray" +"covarPopDistinct" +"COVAR_POPDistinct" +"covarPopForEach" +"COVAR_POPForEach" +"covarPopIf" +"COVAR_POPIf" +"covarPopMap" +"COVAR_POPMap" +"covarPopMatrix" +"covarPopMatrixArgMax" +"covarPopMatrixArgMin" +"covarPopMatrixArray" +"covarPopMatrixDistinct" +"covarPopMatrixForEach" +"covarPopMatrixIf" +"covarPopMatrixMap" +"covarPopMatrixMerge" +"covarPopMatrixNull" +"covarPopMatrixOrDefault" +"covarPopMatrixOrNull" +"covarPopMatrixResample" +"covarPopMatrixSimpleState" +"covarPopMatrixState" +"covarPopMerge" +"COVAR_POPMerge" +"covarPopNull" +"COVAR_POPNull" +"covarPopOrDefault" +"COVAR_POPOrDefault" +"covarPopOrNull" +"COVAR_POPOrNull" +"covarPopResample" +"COVAR_POPResample" +"covarPopSimpleState" +"COVAR_POPSimpleState" "covarPopStable" +"covarPopStableArgMax" +"covarPopStableArgMin" +"covarPopStableArray" +"covarPopStableDistinct" +"covarPopStableForEach" +"covarPopStableIf" +"covarPopStableMap" +"covarPopStableMerge" +"covarPopStableNull" +"covarPopStableOrDefault" +"covarPopStableOrNull" +"covarPopStableResample" +"covarPopStableSimpleState" +"covarPopStableState" +"covarPopState" +"COVAR_POPState" "covarSamp" "COVAR_SAMP" +"covarSampArgMax" +"COVAR_SAMPArgMax" +"covarSampArgMin" +"COVAR_SAMPArgMin" +"covarSampArray" +"COVAR_SAMPArray" +"covarSampDistinct" +"COVAR_SAMPDistinct" +"covarSampForEach" +"COVAR_SAMPForEach" +"covarSampIf" +"COVAR_SAMPIf" +"covarSampMap" +"COVAR_SAMPMap" +"covarSampMatrix" +"covarSampMatrixArgMax" +"covarSampMatrixArgMin" +"covarSampMatrixArray" +"covarSampMatrixDistinct" +"covarSampMatrixForEach" +"covarSampMatrixIf" +"covarSampMatrixMap" +"covarSampMatrixMerge" +"covarSampMatrixNull" +"covarSampMatrixOrDefault" +"covarSampMatrixOrNull" +"covarSampMatrixResample" +"covarSampMatrixSimpleState" +"covarSampMatrixState" +"covarSampMerge" +"COVAR_SAMPMerge" +"covarSampNull" +"COVAR_SAMPNull" +"covarSampOrDefault" +"COVAR_SAMPOrDefault" +"covarSampOrNull" +"COVAR_SAMPOrNull" +"covarSampResample" +"COVAR_SAMPResample" +"covarSampSimpleState" +"COVAR_SAMPSimpleState" "covarSampStable" +"covarSampStableArgMax" +"covarSampStableArgMin" +"covarSampStableArray" +"covarSampStableDistinct" +"covarSampStableForEach" +"covarSampStableIf" +"covarSampStableMap" +"covarSampStableMerge" +"covarSampStableNull" +"covarSampStableOrDefault" +"covarSampStableOrNull" +"covarSampStableResample" +"covarSampStableSimpleState" +"covarSampStableState" +"covarSampState" +"COVAR_SAMPState" +"cramersV" +"cramersVArgMax" +"cramersVArgMin" +"cramersVArray" +"cramersVBiasCorrected" +"cramersVBiasCorrectedArgMax" +"cramersVBiasCorrectedArgMin" +"cramersVBiasCorrectedArray" +"cramersVBiasCorrectedDistinct" +"cramersVBiasCorrectedForEach" +"cramersVBiasCorrectedIf" +"cramersVBiasCorrectedMap" +"cramersVBiasCorrectedMerge" +"cramersVBiasCorrectedNull" +"cramersVBiasCorrectedOrDefault" +"cramersVBiasCorrectedOrNull" +"cramersVBiasCorrectedResample" +"cramersVBiasCorrectedSimpleState" +"cramersVBiasCorrectedState" +"cramersVDistinct" +"cramersVForEach" +"cramersVIf" +"cramersVMap" +"cramersVMerge" +"cramersVNull" +"cramersVOrDefault" +"cramersVOrNull" +"cramersVResample" +"cramersVSimpleState" +"cramersVState" "CRC32" "CRC32IEEE" "CRC64" "CREATE" +"CREATE POLICY" +"CREATE PROFILE" +"CREATE QUOTA" +"CREATE ROLE" +"CREATE ROW POLICY" +"CREATE SETTINGS PROFILE" +"CREATE TABLE" +"CREATE TEMPORARY TABLE" +"CREATE USER" "CROSS" "CUBE" +"curdate" +"current_database" "currentDatabase" +"current_date" +"CURRENT GRANTS" "currentProfiles" +"CURRENT QUOTA" "currentRoles" +"CURRENT ROLES" +"CURRENT ROW" +"current_schemas" +"currentSchemas" +"current_timestamp" +"CURRENT TRANSACTION" "currentUser" +"CURRENT_USER" +"CURRENTUSER" "cutFragment" "cutIPv6" "cutQueryString" "cutQueryStringAndFragment" "cutToFirstSignificantSubdomain" "cutToFirstSignificantSubdomainCustom" +"cutToFirstSignificantSubdomainCustomRFC" "cutToFirstSignificantSubdomainCustomWithWWW" +"cutToFirstSignificantSubdomainCustomWithWWWRFC" +"cutToFirstSignificantSubdomainRFC" "cutToFirstSignificantSubdomainWithWWW" +"cutToFirstSignificantSubdomainWithWWWRFC" "cutURLParameter" "cutWWW" "D" +"damerauLevenshteinDistance" "DATABASE" "DATABASES" "Date" @@ -269,9 +939,11 @@ "Date32" "DATE_ADD" "DATEADD" +"date_diff" "dateDiff" "DATE_DIFF" "DATEDIFF" +"DATE_FORMAT" "dateName" "DATE_SUB" "DATESUB" @@ -280,12 +952,13 @@ "DateTime64" "dateTime64ToSnowflake" "dateTimeToSnowflake" -"date_trunc" "dateTrunc" +"DATE_TRUNC" "DAY" "DAYOFMONTH" "DAYOFWEEK" "DAYOFYEAR" +"DAYS" "DD" "DEC" "Decimal" @@ -293,28 +966,84 @@ "Decimal256" "Decimal32" "Decimal64" +"decodeHTMLComponent" "decodeURLComponent" +"decodeURLFormComponent" "decodeXMLComponent" "decrypt" "DEDUPLICATE" "DEFAULT" +"DEFAULT DATABASE" "defaultProfiles" +"DEFAULT ROLE" "defaultRoles" "defaultValueOfArgumentType" "defaultValueOfTypeName" -"DELAY" +"DEFINER" +"degrees" "DELETE" "DELETE WHERE" "deltaSum" +"deltaSumArgMax" +"deltaSumArgMin" +"deltaSumArray" +"deltaSumDistinct" +"deltaSumForEach" +"deltaSumIf" +"deltaSumMap" +"deltaSumMerge" +"deltaSumNull" +"deltaSumOrDefault" +"deltaSumOrNull" +"deltaSumResample" +"deltaSumSimpleState" +"deltaSumState" "deltaSumTimestamp" +"deltaSumTimestampArgMax" +"deltaSumTimestampArgMin" +"deltaSumTimestampArray" +"deltaSumTimestampDistinct" +"deltaSumTimestampForEach" +"deltaSumTimestampIf" +"deltaSumTimestampMap" +"deltaSumTimestampMerge" +"deltaSumTimestampNull" +"deltaSumTimestampOrDefault" +"deltaSumTimestampOrNull" +"deltaSumTimestampResample" +"deltaSumTimestampSimpleState" +"deltaSumTimestampState" "demangle" "dense_rank" +"dense_rankArgMax" +"dense_rankArgMin" +"dense_rankArray" +"dense_rankDistinct" +"dense_rankForEach" +"dense_rankIf" +"dense_rankMap" +"dense_rankMerge" +"dense_rankNull" +"dense_rankOrDefault" +"dense_rankOrNull" +"dense_rankResample" +"dense_rankSimpleState" +"dense_rankState" +"DEPENDS ON" "DESC" "DESCENDING" "DESCRIBE" "DETACH" +"DETACH PART" "DETACH PARTITION" +"detectCharset" +"detectLanguage" +"detectLanguageMixed" +"detectLanguageUnknown" +"detectProgrammingLanguage" +"detectTonality" "dictGet" +"dictGetAll" "dictGetChildren" "dictGetDate" "dictGetDateOrDefault" @@ -334,6 +1063,10 @@ "dictGetInt64OrDefault" "dictGetInt8" "dictGetInt8OrDefault" +"dictGetIPv4" +"dictGetIPv4OrDefault" +"dictGetIPv6" +"dictGetIPv6OrDefault" "dictGetOrDefault" "dictGetOrNull" "dictGetString" @@ -353,24 +1086,45 @@ "DICTIONARY" "dictIsIn" "DISK" +"displayName" +"distanceL1" +"distanceL2" +"distanceL2Squared" +"distanceLinf" +"distanceLp" "DISTINCT" -"DISTRIBUTED" +"DISTINCT ON" +"DIV" "divide" +"divideDecimal" "domain" +"domainRFC" "domainWithoutWWW" +"domainWithoutWWWRFC" +"dotProduct" "DOUBLE" "DOUBLE PRECISION" +"DOUBLE_SHA1_HASH" +"DOUBLE_SHA1_PASSWORD" "DROP" "DROP COLUMN" "DROP CONSTRAINT" +"DROP DEFAULT" "DROP DETACHED PART" "DROP DETACHED PARTITION" "DROP INDEX" +"DROP PART" "DROP PARTITION" +"DROP PROJECTION" +"DROP STATISTIC" +"DROP TABLE" +"DROP TEMPORARY TABLE" "dumpColumnStructure" "e" +"editDistance" "ELSE" "empty" +"EMPTY" "emptyArrayDate" "emptyArrayDateTime" "emptyArrayFloat32" @@ -385,24 +1139,55 @@ "emptyArrayUInt32" "emptyArrayUInt64" "emptyArrayUInt8" +"EMPTY AS" "enabledProfiles" "enabledRoles" +"ENABLED ROLES" +"encodeURLComponent" +"encodeURLFormComponent" "encodeXMLComponent" "encrypt" "END" "endsWith" +"endsWithUTF8" +"ENFORCED" "ENGINE" "entropy" +"entropyArgMax" +"entropyArgMin" +"entropyArray" +"entropyDistinct" +"entropyForEach" +"entropyIf" +"entropyMap" +"entropyMerge" +"entropyNull" +"entropyOrDefault" +"entropyOrNull" +"entropyResample" +"entropySimpleState" +"entropyState" "Enum" "ENUM" "Enum16" "Enum8" +"EPHEMERAL" +"EPHEMERAL SEQUENTIAL" "equals" "erf" "erfc" "errorCodeToName" +"ESTIMATE" "evalMLMethod" +"EVENT" "EVENTS" +"EVERY" +"EXCEPT" +"EXCEPT DATABASE" +"EXCEPT DATABASES" +"EXCEPT TABLE" +"EXCEPT TABLES" +"EXCHANGE DICTIONARIES" "EXCHANGE TABLES" "EXISTS" "exp" @@ -410,7 +1195,83 @@ "exp2" "EXPLAIN" "exponentialMovingAverage" +"exponentialMovingAverageArgMax" +"exponentialMovingAverageArgMin" +"exponentialMovingAverageArray" +"exponentialMovingAverageDistinct" +"exponentialMovingAverageForEach" +"exponentialMovingAverageIf" +"exponentialMovingAverageMap" +"exponentialMovingAverageMerge" +"exponentialMovingAverageNull" +"exponentialMovingAverageOrDefault" +"exponentialMovingAverageOrNull" +"exponentialMovingAverageResample" +"exponentialMovingAverageSimpleState" +"exponentialMovingAverageState" +"exponentialTimeDecayedAvg" +"exponentialTimeDecayedAvgArgMax" +"exponentialTimeDecayedAvgArgMin" +"exponentialTimeDecayedAvgArray" +"exponentialTimeDecayedAvgDistinct" +"exponentialTimeDecayedAvgForEach" +"exponentialTimeDecayedAvgIf" +"exponentialTimeDecayedAvgMap" +"exponentialTimeDecayedAvgMerge" +"exponentialTimeDecayedAvgNull" +"exponentialTimeDecayedAvgOrDefault" +"exponentialTimeDecayedAvgOrNull" +"exponentialTimeDecayedAvgResample" +"exponentialTimeDecayedAvgSimpleState" +"exponentialTimeDecayedAvgState" +"exponentialTimeDecayedCount" +"exponentialTimeDecayedCountArgMax" +"exponentialTimeDecayedCountArgMin" +"exponentialTimeDecayedCountArray" +"exponentialTimeDecayedCountDistinct" +"exponentialTimeDecayedCountForEach" +"exponentialTimeDecayedCountIf" +"exponentialTimeDecayedCountMap" +"exponentialTimeDecayedCountMerge" +"exponentialTimeDecayedCountNull" +"exponentialTimeDecayedCountOrDefault" +"exponentialTimeDecayedCountOrNull" +"exponentialTimeDecayedCountResample" +"exponentialTimeDecayedCountSimpleState" +"exponentialTimeDecayedCountState" +"exponentialTimeDecayedMax" +"exponentialTimeDecayedMaxArgMax" +"exponentialTimeDecayedMaxArgMin" +"exponentialTimeDecayedMaxArray" +"exponentialTimeDecayedMaxDistinct" +"exponentialTimeDecayedMaxForEach" +"exponentialTimeDecayedMaxIf" +"exponentialTimeDecayedMaxMap" +"exponentialTimeDecayedMaxMerge" +"exponentialTimeDecayedMaxNull" +"exponentialTimeDecayedMaxOrDefault" +"exponentialTimeDecayedMaxOrNull" +"exponentialTimeDecayedMaxResample" +"exponentialTimeDecayedMaxSimpleState" +"exponentialTimeDecayedMaxState" +"exponentialTimeDecayedSum" +"exponentialTimeDecayedSumArgMax" +"exponentialTimeDecayedSumArgMin" +"exponentialTimeDecayedSumArray" +"exponentialTimeDecayedSumDistinct" +"exponentialTimeDecayedSumForEach" +"exponentialTimeDecayedSumIf" +"exponentialTimeDecayedSumMap" +"exponentialTimeDecayedSumMerge" +"exponentialTimeDecayedSumNull" +"exponentialTimeDecayedSumOrDefault" +"exponentialTimeDecayedSumOrNull" +"exponentialTimeDecayedSumResample" +"exponentialTimeDecayedSumSimpleState" +"exponentialTimeDecayedSumState" "EXPRESSION" +"EXTENDED" +"EXTERNAL DDL FROM" "extract" "EXTRACT" "extractAll" @@ -418,40 +1279,106 @@ "extractAllGroupsHorizontal" "extractAllGroupsVertical" "extractGroups" +"extractKeyValuePairs" +"extractKeyValuePairsWithEscaping" "extractTextFromHTML" "extractURLParameter" "extractURLParameterNames" "extractURLParameters" +"factorial" +"FALSE" "farmFingerprint64" "farmHash64" -"FETCHES" +"FETCH" "FETCH PART" "FETCH PARTITION" +"FIELDS" "file" +"FILE" "filesystemAvailable" +"FILESYSTEM CACHE" +"FILESYSTEM CACHES" "filesystemCapacity" -"filesystemFree" +"filesystemUnreserved" +"FILTER" "FINAL" "finalizeAggregation" "FIRST" +"firstLine" "firstSignificantSubdomain" "firstSignificantSubdomainCustom" +"firstSignificantSubdomainCustomRFC" +"firstSignificantSubdomainRFC" "first_value" +"first_valueArgMax" +"first_valueArgMin" +"first_valueArray" +"first_valueDistinct" +"first_valueForEach" +"first_valueIf" +"first_valueMap" +"first_valueMerge" +"first_valueNull" +"first_valueOrDefault" +"first_valueOrNull" +"first_valueResample" +"first_value_respect_nulls" +"first_value_respect_nullsArgMax" +"first_value_respect_nullsArgMin" +"first_value_respect_nullsArray" +"first_value_respect_nullsDistinct" +"first_value_respect_nullsForEach" +"first_value_respect_nullsIf" +"first_value_respect_nullsMap" +"first_value_respect_nullsMerge" +"first_value_respect_nullsNull" +"first_value_respect_nullsOrDefault" +"first_value_respect_nullsOrNull" +"first_value_respect_nullsResample" +"first_value_respect_nullsSimpleState" +"first_value_respect_nullsState" +"first_valueSimpleState" +"first_valueState" "FIXED" "FixedString" +"flameGraph" +"flameGraphArgMax" +"flameGraphArgMin" +"flameGraphArray" +"flameGraphDistinct" +"flameGraphForEach" +"flameGraphIf" +"flameGraphMap" +"flameGraphMerge" +"flameGraphNull" +"flameGraphOrDefault" +"flameGraphOrNull" +"flameGraphResample" +"flameGraphSimpleState" +"flameGraphState" "flatten" +"flattenTuple" "FLOAT" "Float32" "Float64" "floor" -"FLUSH" +"FOLLOWING" "FOR" "ForEach" +"FOREIGN" +"FOREIGN KEY" +"FORGET PARTITION" "format" "FORMAT" +"FORMAT_BYTES" "formatDateTime" -"formatReadableQuantity" +"formatDateTimeInJodaSyntax" +"formatQuery" +"formatQueryOrNull" +"formatQuerySingleLine" +"formatQuerySingleLineOrNull" "formatReadableDecimalSize" +"formatReadableQuantity" "formatReadableSize" "formatReadableTimeDelta" "formatRow" @@ -461,132 +1388,437 @@ "FREEZE" "FROM" "FROM_BASE64" -"toDaysSinceYearZero" +"FROM_DAYS" "fromDaysSinceYearZero" "fromDaysSinceYearZero32" +"FROM INFILE" "fromModifiedJulianDay" "fromModifiedJulianDayOrNull" +"FROM SHARD" "FROM_UNIXTIME" "fromUnixTimestamp" "fromUnixTimestamp64Micro" "fromUnixTimestamp64Milli" "fromUnixTimestamp64Nano" +"fromUnixTimestampInJodaSyntax" +"from_utc_timestamp" +"fromUTCTimestamp" "FULL" "fullHostName" +"FULLTEXT" "FUNCTION" "fuzzBits" "gccMurmurHash" "gcd" +"generateRandomStructure" +"generateULID" "generateUUIDv4" "geoDistance" "geohashDecode" "geohashEncode" "geohashesInBox" +"GEOMETRY" "geoToH3" "geoToS2" "getMacro" +"getOSKernelVersion" "__getScalar" "getServerPort" "getSetting" "getSizeOfEnumType" +"getSubcolumn" +"getTypeSerializationStreams" "GLOBAL" "globalIn" +"GLOBAL IN" "globalInIgnoreSet" "globalNotIn" +"GLOBAL NOT IN" "globalNotInIgnoreSet" "globalNotNullIn" "globalNotNullInIgnoreSet" "globalNullIn" "globalNullInIgnoreSet" "globalVariable" +"GRANT" +"GRANTEES" +"GRANT OPTION FOR" "GRANULARITY" "greatCircleAngle" "greatCircleDistance" "greater" "greaterOrEquals" "greatest" -"GROUP" "groupArray" +"groupArrayArgMax" +"groupArrayArgMin" +"groupArrayArray" +"groupArrayDistinct" +"groupArrayForEach" +"groupArrayIf" "groupArrayInsertAt" +"groupArrayInsertAtArgMax" +"groupArrayInsertAtArgMin" +"groupArrayInsertAtArray" +"groupArrayInsertAtDistinct" +"groupArrayInsertAtForEach" +"groupArrayInsertAtIf" +"groupArrayInsertAtMap" +"groupArrayInsertAtMerge" +"groupArrayInsertAtNull" +"groupArrayInsertAtOrDefault" +"groupArrayInsertAtOrNull" +"groupArrayInsertAtResample" +"groupArrayInsertAtSimpleState" +"groupArrayInsertAtState" +"groupArrayIntersect" +"groupArrayIntersectArgMax" +"groupArrayIntersectArgMin" +"groupArrayIntersectArray" +"groupArrayIntersectDistinct" +"groupArrayIntersectForEach" +"groupArrayIntersectIf" +"groupArrayIntersectMap" +"groupArrayIntersectMerge" +"groupArrayIntersectNull" +"groupArrayIntersectOrDefault" +"groupArrayIntersectOrNull" +"groupArrayIntersectResample" +"groupArrayIntersectSimpleState" +"groupArrayIntersectState" +"groupArrayLast" +"groupArrayLastArgMax" +"groupArrayLastArgMin" +"groupArrayLastArray" +"groupArrayLastDistinct" +"groupArrayLastForEach" +"groupArrayLastIf" +"groupArrayLastMap" +"groupArrayLastMerge" +"groupArrayLastNull" +"groupArrayLastOrDefault" +"groupArrayLastOrNull" +"groupArrayLastResample" +"groupArrayLastSimpleState" +"groupArrayLastState" +"groupArrayMap" +"groupArrayMerge" "groupArrayMovingAvg" +"groupArrayMovingAvgArgMax" +"groupArrayMovingAvgArgMin" +"groupArrayMovingAvgArray" +"groupArrayMovingAvgDistinct" +"groupArrayMovingAvgForEach" +"groupArrayMovingAvgIf" +"groupArrayMovingAvgMap" +"groupArrayMovingAvgMerge" +"groupArrayMovingAvgNull" +"groupArrayMovingAvgOrDefault" +"groupArrayMovingAvgOrNull" +"groupArrayMovingAvgResample" +"groupArrayMovingAvgSimpleState" +"groupArrayMovingAvgState" "groupArrayMovingSum" +"groupArrayMovingSumArgMax" +"groupArrayMovingSumArgMin" +"groupArrayMovingSumArray" +"groupArrayMovingSumDistinct" +"groupArrayMovingSumForEach" +"groupArrayMovingSumIf" +"groupArrayMovingSumMap" +"groupArrayMovingSumMerge" +"groupArrayMovingSumNull" +"groupArrayMovingSumOrDefault" +"groupArrayMovingSumOrNull" +"groupArrayMovingSumResample" +"groupArrayMovingSumSimpleState" +"groupArrayMovingSumState" +"groupArrayNull" +"groupArrayOrDefault" +"groupArrayOrNull" +"groupArrayResample" "groupArraySample" +"groupArraySampleArgMax" +"groupArraySampleArgMin" +"groupArraySampleArray" +"groupArraySampleDistinct" +"groupArraySampleForEach" +"groupArraySampleIf" +"groupArraySampleMap" +"groupArraySampleMerge" +"groupArraySampleNull" +"groupArraySampleOrDefault" +"groupArraySampleOrNull" +"groupArraySampleResample" +"groupArraySampleSimpleState" +"groupArraySampleState" +"groupArraySimpleState" +"groupArraySorted" +"groupArraySortedArgMax" +"groupArraySortedArgMin" +"groupArraySortedArray" +"groupArraySortedDistinct" +"groupArraySortedForEach" +"groupArraySortedIf" +"groupArraySortedMap" +"groupArraySortedMerge" +"groupArraySortedNull" +"groupArraySortedOrDefault" +"groupArraySortedOrNull" +"groupArraySortedResample" +"groupArraySortedSimpleState" +"groupArraySortedState" +"groupArrayState" "groupBitAnd" +"groupBitAndArgMax" +"groupBitAndArgMin" +"groupBitAndArray" +"groupBitAndDistinct" +"groupBitAndForEach" +"groupBitAndIf" +"groupBitAndMap" +"groupBitAndMerge" +"groupBitAndNull" +"groupBitAndOrDefault" +"groupBitAndOrNull" +"groupBitAndResample" +"groupBitAndSimpleState" +"groupBitAndState" "groupBitmap" "groupBitmapAnd" +"groupBitmapAndArgMax" +"groupBitmapAndArgMin" +"groupBitmapAndArray" +"groupBitmapAndDistinct" +"groupBitmapAndForEach" +"groupBitmapAndIf" +"groupBitmapAndMap" +"groupBitmapAndMerge" +"groupBitmapAndNull" +"groupBitmapAndOrDefault" +"groupBitmapAndOrNull" +"groupBitmapAndResample" +"groupBitmapAndSimpleState" +"groupBitmapAndState" +"groupBitmapArgMax" +"groupBitmapArgMin" +"groupBitmapArray" +"groupBitmapDistinct" +"groupBitmapForEach" +"groupBitmapIf" +"groupBitmapMap" +"groupBitmapMerge" +"groupBitmapNull" "groupBitmapOr" +"groupBitmapOrArgMax" +"groupBitmapOrArgMin" +"groupBitmapOrArray" +"groupBitmapOrDefault" +"groupBitmapOrDistinct" +"groupBitmapOrForEach" +"groupBitmapOrIf" +"groupBitmapOrMap" +"groupBitmapOrMerge" +"groupBitmapOrNull" +"groupBitmapOrOrDefault" +"groupBitmapOrOrNull" +"groupBitmapOrResample" +"groupBitmapOrSimpleState" +"groupBitmapOrState" +"groupBitmapResample" +"groupBitmapSimpleState" +"groupBitmapState" "groupBitmapXor" +"groupBitmapXorArgMax" +"groupBitmapXorArgMin" +"groupBitmapXorArray" +"groupBitmapXorDistinct" +"groupBitmapXorForEach" +"groupBitmapXorIf" +"groupBitmapXorMap" +"groupBitmapXorMerge" +"groupBitmapXorNull" +"groupBitmapXorOrDefault" +"groupBitmapXorOrNull" +"groupBitmapXorResample" +"groupBitmapXorSimpleState" +"groupBitmapXorState" "groupBitOr" +"groupBitOrArgMax" +"groupBitOrArgMin" +"groupBitOrArray" +"groupBitOrDistinct" +"groupBitOrForEach" +"groupBitOrIf" +"groupBitOrMap" +"groupBitOrMerge" +"groupBitOrNull" +"groupBitOrOrDefault" +"groupBitOrOrNull" +"groupBitOrResample" +"groupBitOrSimpleState" +"groupBitOrState" "groupBitXor" +"groupBitXorArgMax" +"groupBitXorArgMin" +"groupBitXorArray" +"groupBitXorDistinct" +"groupBitXorForEach" +"groupBitXorIf" +"groupBitXorMap" +"groupBitXorMerge" +"groupBitXorNull" +"groupBitXorOrDefault" +"groupBitXorOrNull" +"groupBitXorResample" +"groupBitXorSimpleState" +"groupBitXorState" "GROUP BY" +"GROUPING SETS" +"GROUPS" "groupUniqArray" +"groupUniqArrayArgMax" +"groupUniqArrayArgMin" +"groupUniqArrayArray" +"groupUniqArrayDistinct" +"groupUniqArrayForEach" +"groupUniqArrayIf" +"groupUniqArrayMap" +"groupUniqArrayMerge" +"groupUniqArrayNull" +"groupUniqArrayOrDefault" +"groupUniqArrayOrNull" +"groupUniqArrayResample" +"groupUniqArraySimpleState" +"groupUniqArrayState" +"H" +"h3CellAreaM2" +"h3CellAreaRads2" +"h3Distance" "h3EdgeAngle" +"h3EdgeLengthKm" "h3EdgeLengthM" +"h3ExactEdgeLengthKm" +"h3ExactEdgeLengthM" +"h3ExactEdgeLengthRads" "h3GetBaseCell" +"h3GetDestinationIndexFromUnidirectionalEdge" "h3GetFaces" +"h3GetIndexesFromUnidirectionalEdge" +"h3GetOriginIndexFromUnidirectionalEdge" +"h3GetPentagonIndexes" +"h3GetRes0Indexes" "h3GetResolution" +"h3GetUnidirectionalEdge" +"h3GetUnidirectionalEdgeBoundary" +"h3GetUnidirectionalEdgesFromHexagon" +"h3HexAreaKm2" "h3HexAreaM2" +"h3HexRing" "h3IndexesAreNeighbors" "h3IsPentagon" "h3IsResClassIII" "h3IsValid" "h3kRing" +"h3Line" +"h3NumHexagons" +"h3PointDistKm" +"h3PointDistM" +"h3PointDistRads" +"h3ToCenterChild" "h3ToChildren" "h3ToGeo" "h3ToGeoBoundary" "h3ToParent" "h3ToString" +"h3UnidirectionalEdgeIsValid" "halfMD5" "has" "hasAll" "hasAny" "hasColumnInTable" +"HASH" +"hasSubsequence" +"hasSubsequenceCaseInsensitive" +"hasSubsequenceCaseInsensitiveUTF8" +"hasSubsequenceUTF8" "hasSubstr" "hasThreadFuzzer" "hasToken" "hasTokenCaseInsensitive" +"hasTokenCaseInsensitiveOrNull" +"hasTokenOrNull" "HAVING" +"HDFS" "hex" "HH" "HIERARCHICAL" "histogram" +"histogramArgMax" +"histogramArgMin" +"histogramArray" +"histogramDistinct" +"histogramForEach" +"histogramIf" +"histogramMap" +"histogramMerge" +"histogramNull" +"histogramOrDefault" +"histogramOrNull" +"histogramResample" +"histogramSimpleState" +"histogramState" "hiveHash" +"hop" +"hopEnd" +"hopStart" +"HOST" "hostname" "hostName" "HOUR" +"HOURS" +"HTTP" "hypot" "ID" +"IDENTIFIED" "identity" +"idnaDecode" +"idnaEncode" "if" -"IF" +"IF EMPTY" "IF EXISTS" "IF NOT EXISTS" "ifNotFinite" "ifNull" "ignore" +"IGNORE NULLS" "ilike" "ILIKE" "in" "IN" "INDEX" +"INDEXES" "indexHint" "indexOf" +"INDICES" "INET4" "INET6" "INET6_ATON" "INET6_NTOA" "INET_ATON" "INET_NTOA" -"INF" +"INHERIT" "inIgnoreSet" +"initcap" +"initcapUTF8" "initializeAggregation" "initial_query_id" "initialQueryID" "INJECTIVE" "INNER" "IN PARTITION" -"INSERT" "INSERT INTO" +"instr" "INT" "INT1" "Int128" @@ -602,12 +1834,31 @@ "INTEGER" "INTEGER SIGNED" "INTEGER UNSIGNED" +"INTERPOLATE" +"INTERSECT" "INTERVAL" "IntervalDay" "IntervalHour" "intervalLengthSum" +"intervalLengthSumArgMax" +"intervalLengthSumArgMin" +"intervalLengthSumArray" +"intervalLengthSumDistinct" +"intervalLengthSumForEach" +"intervalLengthSumIf" +"intervalLengthSumMap" +"intervalLengthSumMerge" +"intervalLengthSumNull" +"intervalLengthSumOrDefault" +"intervalLengthSumOrNull" +"intervalLengthSumResample" +"intervalLengthSumSimpleState" +"intervalLengthSumState" +"IntervalMicrosecond" +"IntervalMillisecond" "IntervalMinute" "IntervalMonth" +"IntervalNanosecond" "IntervalQuarter" "IntervalSecond" "IntervalWeek" @@ -616,21 +1867,26 @@ "intExp2" "intHash32" "intHash64" -"INTO" "INTO OUTFILE" "INT SIGNED" "INT UNSIGNED" +"INVISIBLE" +"INVOKER" +"IP" "IPv4" "IPv4CIDRToRange" "IPv4NumToString" "IPv4NumToStringClassC" "IPv4StringToNum" +"IPv4StringToNumOrDefault" +"IPv4StringToNumOrNull" "IPv4ToIPv6" "IPv6" "IPv6CIDRToRange" "IPv6NumToString" "IPv6StringToNum" -"IS" +"IPv6StringToNumOrDefault" +"IPv6StringToNumOrNull" "isConstant" "isDecimalOverflow" "isFinite" @@ -639,64 +1895,220 @@ "isIPv4String" "isIPv6String" "isNaN" +"isNotDistinctFrom" +"IS NOT DISTINCT FROM" "isNotNull" +"IS NOT NULL" "isNull" +"IS NULL" +"isNullable" "IS_OBJECT_ID" "isValidJSON" "isValidUTF8" "isZeroOrNull" +"jaroSimilarity" +"jaroWinklerSimilarity" "javaHash" "javaHashUTF16LE" "JOIN" "joinGet" "joinGetOrNull" +"JSON" +"JSONArrayLength" +"JSON_ARRAY_LENGTH" "JSON_EXISTS" "JSONExtract" "JSONExtractArrayRaw" "JSONExtractBool" "JSONExtractFloat" "JSONExtractInt" +"JSONExtractKeys" "JSONExtractKeysAndValues" "JSONExtractKeysAndValuesRaw" -"JSONExtractKeys" "JSONExtractRaw" "JSONExtractString" "JSONExtractUInt" "JSONHas" "JSONKey" "JSONLength" +"jsonMergePatch" "JSON_QUERY" "JSONType" "JSON_VALUE" "jumpConsistentHash" +"kafkaMurmurHash" +"KERBEROS" "KEY" -# Key words (based on keywords from antlr parser) +"KEY BY" +"KEYED BY" +"KEYS" "KILL" +"KIND" +"kolmogorovSmirnovTest" +"kolmogorovSmirnovTestArgMax" +"kolmogorovSmirnovTestArgMin" +"kolmogorovSmirnovTestArray" +"kolmogorovSmirnovTestDistinct" +"kolmogorovSmirnovTestForEach" +"kolmogorovSmirnovTestIf" +"kolmogorovSmirnovTestMap" +"kolmogorovSmirnovTestMerge" +"kolmogorovSmirnovTestNull" +"kolmogorovSmirnovTestOrDefault" +"kolmogorovSmirnovTestOrNull" +"kolmogorovSmirnovTestResample" +"kolmogorovSmirnovTestSimpleState" +"kolmogorovSmirnovTestState" +"kostikConsistentHash" +"kql_array_sort_asc" +"kql_array_sort_desc" "kurtPop" +"kurtPopArgMax" +"kurtPopArgMin" +"kurtPopArray" +"kurtPopDistinct" +"kurtPopForEach" +"kurtPopIf" +"kurtPopMap" +"kurtPopMerge" +"kurtPopNull" +"kurtPopOrDefault" +"kurtPopOrNull" +"kurtPopResample" +"kurtPopSimpleState" +"kurtPopState" "kurtSamp" +"kurtSampArgMax" +"kurtSampArgMin" +"kurtSampArray" +"kurtSampDistinct" +"kurtSampForEach" +"kurtSampIf" +"kurtSampMap" +"kurtSampMerge" +"kurtSampNull" +"kurtSampOrDefault" +"kurtSampOrNull" +"kurtSampResample" +"kurtSampSimpleState" +"kurtSampState" +"L1Distance" +"L1Norm" +"L1Normalize" +"L2Distance" +"L2Norm" +"L2Normalize" +"L2SquaredDistance" +"L2SquaredNorm" "lagInFrame" +"lagInFrameArgMax" +"lagInFrameArgMin" +"lagInFrameArray" +"lagInFrameDistinct" +"lagInFrameForEach" +"lagInFrameIf" +"lagInFrameMap" +"lagInFrameMerge" +"lagInFrameNull" +"lagInFrameOrDefault" +"lagInFrameOrNull" +"lagInFrameResample" +"lagInFrameSimpleState" +"lagInFrameState" +"LARGE OBJECT" +"largestTriangleThreeBuckets" +"largestTriangleThreeBucketsArgMax" +"largestTriangleThreeBucketsArgMin" +"largestTriangleThreeBucketsArray" +"largestTriangleThreeBucketsDistinct" +"largestTriangleThreeBucketsForEach" +"largestTriangleThreeBucketsIf" +"largestTriangleThreeBucketsMap" +"largestTriangleThreeBucketsMerge" +"largestTriangleThreeBucketsNull" +"largestTriangleThreeBucketsOrDefault" +"largestTriangleThreeBucketsOrNull" +"largestTriangleThreeBucketsResample" +"largestTriangleThreeBucketsSimpleState" +"largestTriangleThreeBucketsState" "LAST" +"LAST_DAY" "last_value" +"last_valueArgMax" +"last_valueArgMin" +"last_valueArray" +"last_valueDistinct" +"last_valueForEach" +"last_valueIf" +"last_valueMap" +"last_valueMerge" +"last_valueNull" +"last_valueOrDefault" +"last_valueOrNull" +"last_valueResample" +"last_value_respect_nulls" +"last_value_respect_nullsArgMax" +"last_value_respect_nullsArgMin" +"last_value_respect_nullsArray" +"last_value_respect_nullsDistinct" +"last_value_respect_nullsForEach" +"last_value_respect_nullsIf" +"last_value_respect_nullsMap" +"last_value_respect_nullsMerge" +"last_value_respect_nullsNull" +"last_value_respect_nullsOrDefault" +"last_value_respect_nullsOrNull" +"last_value_respect_nullsResample" +"last_value_respect_nullsSimpleState" +"last_value_respect_nullsState" +"last_valueSimpleState" +"last_valueState" "LAYOUT" "lcase" "lcm" +"LDAP" "leadInFrame" +"leadInFrameArgMax" +"leadInFrameArgMin" +"leadInFrameArray" +"leadInFrameDistinct" +"leadInFrameForEach" +"leadInFrameIf" +"leadInFrameMap" +"leadInFrameMerge" +"leadInFrameNull" +"leadInFrameOrDefault" +"leadInFrameOrNull" +"leadInFrameResample" +"leadInFrameSimpleState" +"leadInFrameState" "LEADING" "least" +"left" "LEFT" "LEFT ARRAY JOIN" "leftPad" "leftPadUTF8" +"leftUTF8" "lemmatize" "length" "lengthUTF8" "less" "lessOrEquals" +"LESS THAN" +"LEVEL" +"levenshteinDistance" "lgamma" "LIFETIME" +"LIGHTWEIGHT" "like" "LIKE" "LIMIT" +"LINEAR" +"LinfDistance" +"LinfNorm" +"LinfNormalize" +"LIST" "LIVE" "ln" "LOCAL" @@ -705,7 +2117,6 @@ "log10" "log1p" "log2" -"LOGS" "logTrace" "LONGBLOB" "LONGTEXT" @@ -715,76 +2126,467 @@ "lower" "lowerUTF8" "lpad" +"LpDistance" +"LpNorm" +"LpNormalize" +"ltrim" "LTRIM" +"lttb" +"lttbArgMax" +"lttbArgMin" +"lttbArray" +"lttbDistinct" +"lttbForEach" +"lttbIf" +"lttbMap" +"lttbMerge" +"lttbNull" +"lttbOrDefault" +"lttbOrNull" +"lttbResample" +"lttbSimpleState" +"lttbState" "M" "MACNumToString" "MACStringToNum" "MACStringToOUI" +"makeDate" +"makeDate32" +"makeDateTime" +"makeDateTime64" "mannWhitneyUTest" +"mannWhitneyUTestArgMax" +"mannWhitneyUTestArgMin" +"mannWhitneyUTestArray" +"mannWhitneyUTestDistinct" +"mannWhitneyUTestForEach" +"mannWhitneyUTestIf" +"mannWhitneyUTestMap" +"mannWhitneyUTestMerge" +"mannWhitneyUTestNull" +"mannWhitneyUTestOrDefault" +"mannWhitneyUTestOrNull" +"mannWhitneyUTestResample" +"mannWhitneyUTestSimpleState" +"mannWhitneyUTestState" "map" "Map" "mapAdd" +"mapAll" +"mapApply" +"mapConcat" "mapContains" +"mapContainsKeyLike" +"mapExists" +"mapExtractKeyLike" +"mapFilter" +"mapFromArrays" +"MAP_FROM_ARRAYS" +"mapFromString" "mapKeys" +"mapPartialReverseSort" +"mapPartialSort" "mapPopulateSeries" +"mapReverseSort" +"mapSort" "mapSubtract" +"mapUpdate" "mapValues" "match" +"MATCH" "materialize" "MATERIALIZE" +"MATERIALIZE COLUMN" "MATERIALIZED" "MATERIALIZE INDEX" +"MATERIALIZE PROJECTION" +"MATERIALIZE STATISTIC" "MATERIALIZE TTL" "max" "MAX" +"max2" +"maxArgMax" +"maxArgMin" +"maxArray" +"maxDistinct" +"maxForEach" +"maxIf" "maxIntersections" +"maxIntersectionsArgMax" +"maxIntersectionsArgMin" +"maxIntersectionsArray" +"maxIntersectionsDistinct" +"maxIntersectionsForEach" +"maxIntersectionsIf" +"maxIntersectionsMap" +"maxIntersectionsMerge" +"maxIntersectionsNull" +"maxIntersectionsOrDefault" +"maxIntersectionsOrNull" "maxIntersectionsPosition" +"maxIntersectionsPositionArgMax" +"maxIntersectionsPositionArgMin" +"maxIntersectionsPositionArray" +"maxIntersectionsPositionDistinct" +"maxIntersectionsPositionForEach" +"maxIntersectionsPositionIf" +"maxIntersectionsPositionMap" +"maxIntersectionsPositionMerge" +"maxIntersectionsPositionNull" +"maxIntersectionsPositionOrDefault" +"maxIntersectionsPositionOrNull" +"maxIntersectionsPositionResample" +"maxIntersectionsPositionSimpleState" +"maxIntersectionsPositionState" +"maxIntersectionsResample" +"maxIntersectionsSimpleState" +"maxIntersectionsState" "maxMap" +"maxMappedArrays" +"maxMappedArraysArgMax" +"maxMappedArraysArgMin" +"maxMappedArraysArray" +"maxMappedArraysDistinct" +"maxMappedArraysForEach" +"maxMappedArraysIf" +"maxMappedArraysMap" +"maxMappedArraysMerge" +"maxMappedArraysNull" +"maxMappedArraysOrDefault" +"maxMappedArraysOrNull" +"maxMappedArraysResample" +"maxMappedArraysSimpleState" +"maxMappedArraysState" +"maxMerge" +"maxNull" +"maxOrDefault" +"maxOrNull" +"maxResample" +"maxSimpleState" +"maxState" +"MCS" "MD4" "MD5" +"meanZTest" +"meanZTestArgMax" +"meanZTestArgMin" +"meanZTestArray" +"meanZTestDistinct" +"meanZTestForEach" +"meanZTestIf" +"meanZTestMap" +"meanZTestMerge" +"meanZTestNull" +"meanZTestOrDefault" +"meanZTestOrNull" +"meanZTestResample" +"meanZTestSimpleState" +"meanZTestState" "median" +"medianArgMax" +"medianArgMin" +"medianArray" "medianBFloat16" +"medianBFloat16ArgMax" +"medianBFloat16ArgMin" +"medianBFloat16Array" +"medianBFloat16Distinct" +"medianBFloat16ForEach" +"medianBFloat16If" +"medianBFloat16Map" +"medianBFloat16Merge" +"medianBFloat16Null" +"medianBFloat16OrDefault" +"medianBFloat16OrNull" +"medianBFloat16Resample" +"medianBFloat16SimpleState" +"medianBFloat16State" "medianBFloat16Weighted" +"medianBFloat16WeightedArgMax" +"medianBFloat16WeightedArgMin" +"medianBFloat16WeightedArray" +"medianBFloat16WeightedDistinct" +"medianBFloat16WeightedForEach" +"medianBFloat16WeightedIf" +"medianBFloat16WeightedMap" +"medianBFloat16WeightedMerge" +"medianBFloat16WeightedNull" +"medianBFloat16WeightedOrDefault" +"medianBFloat16WeightedOrNull" +"medianBFloat16WeightedResample" +"medianBFloat16WeightedSimpleState" +"medianBFloat16WeightedState" +"medianDD" +"medianDDArgMax" +"medianDDArgMin" +"medianDDArray" +"medianDDDistinct" +"medianDDForEach" +"medianDDIf" +"medianDDMap" +"medianDDMerge" +"medianDDNull" +"medianDDOrDefault" +"medianDDOrNull" +"medianDDResample" +"medianDDSimpleState" +"medianDDState" "medianDeterministic" +"medianDeterministicArgMax" +"medianDeterministicArgMin" +"medianDeterministicArray" +"medianDeterministicDistinct" +"medianDeterministicForEach" +"medianDeterministicIf" +"medianDeterministicMap" +"medianDeterministicMerge" +"medianDeterministicNull" +"medianDeterministicOrDefault" +"medianDeterministicOrNull" +"medianDeterministicResample" +"medianDeterministicSimpleState" +"medianDeterministicState" +"medianDistinct" "medianExact" +"medianExactArgMax" +"medianExactArgMin" +"medianExactArray" +"medianExactDistinct" +"medianExactForEach" "medianExactHigh" +"medianExactHighArgMax" +"medianExactHighArgMin" +"medianExactHighArray" +"medianExactHighDistinct" +"medianExactHighForEach" +"medianExactHighIf" +"medianExactHighMap" +"medianExactHighMerge" +"medianExactHighNull" +"medianExactHighOrDefault" +"medianExactHighOrNull" +"medianExactHighResample" +"medianExactHighSimpleState" +"medianExactHighState" +"medianExactIf" "medianExactLow" +"medianExactLowArgMax" +"medianExactLowArgMin" +"medianExactLowArray" +"medianExactLowDistinct" +"medianExactLowForEach" +"medianExactLowIf" +"medianExactLowMap" +"medianExactLowMerge" +"medianExactLowNull" +"medianExactLowOrDefault" +"medianExactLowOrNull" +"medianExactLowResample" +"medianExactLowSimpleState" +"medianExactLowState" +"medianExactMap" +"medianExactMerge" +"medianExactNull" +"medianExactOrDefault" +"medianExactOrNull" +"medianExactResample" +"medianExactSimpleState" +"medianExactState" "medianExactWeighted" +"medianExactWeightedArgMax" +"medianExactWeightedArgMin" +"medianExactWeightedArray" +"medianExactWeightedDistinct" +"medianExactWeightedForEach" +"medianExactWeightedIf" +"medianExactWeightedMap" +"medianExactWeightedMerge" +"medianExactWeightedNull" +"medianExactWeightedOrDefault" +"medianExactWeightedOrNull" +"medianExactWeightedResample" +"medianExactWeightedSimpleState" +"medianExactWeightedState" +"medianForEach" +"medianGK" +"medianGKArgMax" +"medianGKArgMin" +"medianGKArray" +"medianGKDistinct" +"medianGKForEach" +"medianGKIf" +"medianGKMap" +"medianGKMerge" +"medianGKNull" +"medianGKOrDefault" +"medianGKOrNull" +"medianGKResample" +"medianGKSimpleState" +"medianGKState" +"medianIf" +"medianInterpolatedWeighted" +"medianInterpolatedWeightedArgMax" +"medianInterpolatedWeightedArgMin" +"medianInterpolatedWeightedArray" +"medianInterpolatedWeightedDistinct" +"medianInterpolatedWeightedForEach" +"medianInterpolatedWeightedIf" +"medianInterpolatedWeightedMap" +"medianInterpolatedWeightedMerge" +"medianInterpolatedWeightedNull" +"medianInterpolatedWeightedOrDefault" +"medianInterpolatedWeightedOrNull" +"medianInterpolatedWeightedResample" +"medianInterpolatedWeightedSimpleState" +"medianInterpolatedWeightedState" +"medianMap" +"medianMerge" +"medianNull" +"medianOrDefault" +"medianOrNull" +"medianResample" +"medianSimpleState" +"medianState" "medianTDigest" +"medianTDigestArgMax" +"medianTDigestArgMin" +"medianTDigestArray" +"medianTDigestDistinct" +"medianTDigestForEach" +"medianTDigestIf" +"medianTDigestMap" +"medianTDigestMerge" +"medianTDigestNull" +"medianTDigestOrDefault" +"medianTDigestOrNull" +"medianTDigestResample" +"medianTDigestSimpleState" +"medianTDigestState" "medianTDigestWeighted" +"medianTDigestWeightedArgMax" +"medianTDigestWeightedArgMin" +"medianTDigestWeightedArray" +"medianTDigestWeightedDistinct" +"medianTDigestWeightedForEach" +"medianTDigestWeightedIf" +"medianTDigestWeightedMap" +"medianTDigestWeightedMerge" +"medianTDigestWeightedNull" +"medianTDigestWeightedOrDefault" +"medianTDigestWeightedOrNull" +"medianTDigestWeightedResample" +"medianTDigestWeightedSimpleState" +"medianTDigestWeightedState" "medianTiming" +"medianTimingArgMax" +"medianTimingArgMin" +"medianTimingArray" +"medianTimingDistinct" +"medianTimingForEach" +"medianTimingIf" +"medianTimingMap" +"medianTimingMerge" +"medianTimingNull" +"medianTimingOrDefault" +"medianTimingOrNull" +"medianTimingResample" +"medianTimingSimpleState" +"medianTimingState" "medianTimingWeighted" +"medianTimingWeightedArgMax" +"medianTimingWeightedArgMin" +"medianTimingWeightedArray" +"medianTimingWeightedDistinct" +"medianTimingWeightedForEach" +"medianTimingWeightedIf" +"medianTimingWeightedMap" +"medianTimingWeightedMerge" +"medianTimingWeightedNull" +"medianTimingWeightedOrDefault" +"medianTimingWeightedOrNull" +"medianTimingWeightedResample" +"medianTimingWeightedSimpleState" +"medianTimingWeightedState" "MEDIUMBLOB" "MEDIUMINT" "MEDIUMINT SIGNED" "MEDIUMINT UNSIGNED" "MEDIUMTEXT" +"MEMORY" "Merge" "MERGES" "metroHash64" "MI" +"MICROSECOND" +"MICROSECONDS" "mid" +"MILLISECOND" +"MILLISECONDS" "min" "MIN" +"min2" +"minArgMax" +"minArgMin" +"minArray" +"minDistinct" +"minForEach" +"minIf" "minMap" +"minMappedArrays" +"minMappedArraysArgMax" +"minMappedArraysArgMin" +"minMappedArraysArray" +"minMappedArraysDistinct" +"minMappedArraysForEach" +"minMappedArraysIf" +"minMappedArraysMap" +"minMappedArraysMerge" +"minMappedArraysNull" +"minMappedArraysOrDefault" +"minMappedArraysOrNull" +"minMappedArraysResample" +"minMappedArraysSimpleState" +"minMappedArraysState" +"minMerge" +"minNull" +"minOrDefault" +"minOrNull" +"minResample" +"minSampleSizeContinous" +"minSampleSizeContinuous" +"minSampleSizeConversion" +"minSimpleState" +"minState" "minus" "MINUTE" +"MINUTES" +"mismatches" "MM" "mod" +"MOD" "MODIFY" "MODIFY COLUMN" +"MODIFY COMMENT" "MODIFY ORDER BY" "MODIFY QUERY" +"MODIFY REFRESH" +"MODIFY SAMPLE BY" "MODIFY SETTING" +"MODIFY SQL SECURITY" "MODIFY TTL" "modulo" "moduloLegacy" "moduloOrZero" "MONTH" +"monthName" +"MONTHS" +"mortonDecode" +"mortonEncode" "MOVE" "MOVE PART" "MOVE PARTITION" "movingXXX" +"MS" "multiFuzzyMatchAllIndices" "multiFuzzyMatchAny" "multiFuzzyMatchAnyIndex" @@ -793,6 +2595,7 @@ "multiMatchAny" "multiMatchAnyIndex" "multiply" +"multiplyDecimal" "MultiPolygon" "multiSearchAllPositions" "multiSearchAllPositionsCaseInsensitive" @@ -818,7 +2621,9 @@ "MUTATION" "N" "NAME" -"NAN_SQL" +"NAMED COLLECTION" +"NANOSECOND" +"NANOSECONDS" "NATIONAL CHAR" "NATIONAL CHARACTER" "NATIONAL CHARACTER LARGE OBJECT" @@ -829,8 +2634,10 @@ "NCHAR VARYING" "negate" "neighbor" +"nested" "Nested" "netloc" +"NEXT" "ngramDistance" "ngramDistanceCaseInsensitive" "ngramDistanceCaseInsensitiveUTF8" @@ -843,6 +2650,7 @@ "ngramMinHashCaseInsensitive" "ngramMinHashCaseInsensitiveUTF8" "ngramMinHashUTF8" +"ngrams" "ngramSearch" "ngramSearchCaseInsensitive" "ngramSearchCaseInsensitiveUTF8" @@ -851,68 +2659,207 @@ "ngramSimHashCaseInsensitive" "ngramSimHashCaseInsensitiveUTF8" "ngramSimHashUTF8" -"NO" +"NO ACTION" "NO DELAY" +"NO LIMITS" "NONE" +"nonNegativeDerivative" +"nonNegativeDerivativeArgMax" +"nonNegativeDerivativeArgMin" +"nonNegativeDerivativeArray" +"nonNegativeDerivativeDistinct" +"nonNegativeDerivativeForEach" +"nonNegativeDerivativeIf" +"nonNegativeDerivativeMap" +"nonNegativeDerivativeMerge" +"nonNegativeDerivativeNull" +"nonNegativeDerivativeOrDefault" +"nonNegativeDerivativeOrNull" +"nonNegativeDerivativeResample" +"nonNegativeDerivativeSimpleState" +"nonNegativeDerivativeState" +"NO_PASSWORD" "normalizedQueryHash" "normalizedQueryHashKeepNames" +"normalizeL1" +"normalizeL2" +"normalizeLinf" +"normalizeLp" "normalizeQuery" "normalizeQueryKeepNames" +"normalizeUTF8NFC" +"normalizeUTF8NFD" +"normalizeUTF8NFKC" +"normalizeUTF8NFKD" +"normL1" +"normL2" +"normL2Squared" +"normLinf" +"normLp" "not" "NOT" +"NOT BETWEEN" "notEmpty" "notEquals" "nothing" "Nothing" +"nothingArgMax" +"nothingArgMin" +"nothingArray" +"nothingDistinct" +"nothingForEach" +"nothingIf" +"nothingMap" +"nothingMerge" +"nothingNull" +"nothingNullArgMax" +"nothingNullArgMin" +"nothingNullArray" +"nothingNullDistinct" +"nothingNullForEach" +"nothingNullIf" +"nothingNullMap" +"nothingNullMerge" +"nothingNullNull" +"nothingNullOrDefault" +"nothingNullOrNull" +"nothingNullResample" +"nothingNullSimpleState" +"nothingNullState" +"nothingOrDefault" +"nothingOrNull" +"nothingResample" +"nothingSimpleState" +"nothingState" +"nothingUInt64" +"nothingUInt64ArgMax" +"nothingUInt64ArgMin" +"nothingUInt64Array" +"nothingUInt64Distinct" +"nothingUInt64ForEach" +"nothingUInt64If" +"nothingUInt64Map" +"nothingUInt64Merge" +"nothingUInt64Null" +"nothingUInt64OrDefault" +"nothingUInt64OrNull" +"nothingUInt64Resample" +"nothingUInt64SimpleState" +"nothingUInt64State" +"NOT IDENTIFIED" "notILike" +"NOT ILIKE" "notIn" +"NOT IN" "notInIgnoreSet" +"NOT KEYED" "notLike" +"NOT LIKE" "notNullIn" "notNullInIgnoreSet" +"NOT OVERRIDABLE" "now" "now64" +"nowInBlock" +"NS" +"nth_value" +"nth_valueArgMax" +"nth_valueArgMin" +"nth_valueArray" +"nth_valueDistinct" +"nth_valueForEach" +"nth_valueIf" +"nth_valueMap" +"nth_valueMerge" +"nth_valueNull" +"nth_valueOrDefault" +"nth_valueOrNull" +"nth_valueResample" +"nth_valueSimpleState" +"nth_valueState" +"ntile" +"ntileArgMax" +"ntileArgMin" +"ntileArray" +"ntileDistinct" +"ntileForEach" +"ntileIf" +"ntileMap" +"ntileMerge" +"ntileNull" +"ntileOrDefault" +"ntileOrNull" +"ntileResample" +"ntileSimpleState" +"ntileState" "Null" +"NULL" "Nullable" "nullIf" "nullIn" "nullInIgnoreSet" "NULLS" -"NULL_SQL" "NUMERIC" "NVARCHAR" +"Object" +"OCTET_LENGTH" "OFFSET" "ON" +"ON DELETE" "ONLY" -"OPTIMIZE" +"ON UPDATE" +"ON VOLUME" "OPTIMIZE TABLE" "or" "OR" -"ORDER" "ORDER BY" "OR REPLACE" "OUTER" -"OUTFILE" +"OVER" +"OVERRIDABLE" +"parseDateTime" "parseDateTime32BestEffort" "parseDateTime32BestEffortOrNull" "parseDateTime32BestEffortOrZero" "parseDateTime64BestEffort" "parseDateTime64BestEffortOrNull" "parseDateTime64BestEffortOrZero" +"parseDateTime64BestEffortUS" +"parseDateTime64BestEffortUSOrNull" +"parseDateTime64BestEffortUSOrZero" "parseDateTimeBestEffort" "parseDateTimeBestEffortOrNull" "parseDateTimeBestEffortOrZero" "parseDateTimeBestEffortUS" "parseDateTimeBestEffortUSOrNull" "parseDateTimeBestEffortUSOrZero" +"parseDateTimeInJodaSyntax" +"parseDateTimeInJodaSyntaxOrNull" +"parseDateTimeInJodaSyntaxOrZero" +"parseDateTimeOrNull" +"parseDateTimeOrZero" "parseTimeDelta" +"PART" +"PARTIAL" "PARTITION" "PARTITION BY" "partitionId" +"PARTITIONS" +"PART_MOVE_TO_SHARD" +"PASTE" "path" "pathFull" +"PERIODIC REFRESH" +"PERMANENTLY" +"PERMISSIVE" +"PERSISTENT" +"PERSISTENT SEQUENTIAL" "pi" +"PIPELINE" +"PLAINTEXT_PASSWORD" +"PLAN" "plus" +"pmod" "Point" "pointInEllipses" "pointInPolygon" @@ -935,71 +2882,620 @@ "polygonsWithinSpherical" "POPULATE" "port" +"portRFC" "position" "positionCaseInsensitive" "positionCaseInsensitiveUTF8" "positionUTF8" +"positive_modulo" +"positiveModulo" "pow" "power" +"PRECEDING" +"PRECISION" "PREWHERE" "PRIMARY" "PRIMARY KEY" +"PROFILE" "PROJECTION" +"proportionsZTest" +"Protobuf" "protocol" +"PULL" +"punycodeDecode" +"punycodeEncode" "Q" "QQ" "quantile" +"quantileArgMax" +"quantileArgMin" +"quantileArray" "quantileBFloat16" +"quantileBFloat16ArgMax" +"quantileBFloat16ArgMin" +"quantileBFloat16Array" +"quantileBFloat16Distinct" +"quantileBFloat16ForEach" +"quantileBFloat16If" +"quantileBFloat16Map" +"quantileBFloat16Merge" +"quantileBFloat16Null" +"quantileBFloat16OrDefault" +"quantileBFloat16OrNull" +"quantileBFloat16Resample" +"quantileBFloat16SimpleState" +"quantileBFloat16State" "quantileBFloat16Weighted" +"quantileBFloat16WeightedArgMax" +"quantileBFloat16WeightedArgMin" +"quantileBFloat16WeightedArray" +"quantileBFloat16WeightedDistinct" +"quantileBFloat16WeightedForEach" +"quantileBFloat16WeightedIf" +"quantileBFloat16WeightedMap" +"quantileBFloat16WeightedMerge" +"quantileBFloat16WeightedNull" +"quantileBFloat16WeightedOrDefault" +"quantileBFloat16WeightedOrNull" +"quantileBFloat16WeightedResample" +"quantileBFloat16WeightedSimpleState" +"quantileBFloat16WeightedState" +"quantileDD" +"quantileDDArgMax" +"quantileDDArgMin" +"quantileDDArray" +"quantileDDDistinct" +"quantileDDForEach" +"quantileDDIf" +"quantileDDMap" +"quantileDDMerge" +"quantileDDNull" +"quantileDDOrDefault" +"quantileDDOrNull" +"quantileDDResample" +"quantileDDSimpleState" +"quantileDDState" "quantileDeterministic" +"quantileDeterministicArgMax" +"quantileDeterministicArgMin" +"quantileDeterministicArray" +"quantileDeterministicDistinct" +"quantileDeterministicForEach" +"quantileDeterministicIf" +"quantileDeterministicMap" +"quantileDeterministicMerge" +"quantileDeterministicNull" +"quantileDeterministicOrDefault" +"quantileDeterministicOrNull" +"quantileDeterministicResample" +"quantileDeterministicSimpleState" +"quantileDeterministicState" +"quantileDistinct" "quantileExact" +"quantileExactArgMax" +"quantileExactArgMin" +"quantileExactArray" +"quantileExactDistinct" "quantileExactExclusive" +"quantileExactExclusiveArgMax" +"quantileExactExclusiveArgMin" +"quantileExactExclusiveArray" +"quantileExactExclusiveDistinct" +"quantileExactExclusiveForEach" +"quantileExactExclusiveIf" +"quantileExactExclusiveMap" +"quantileExactExclusiveMerge" +"quantileExactExclusiveNull" +"quantileExactExclusiveOrDefault" +"quantileExactExclusiveOrNull" +"quantileExactExclusiveResample" +"quantileExactExclusiveSimpleState" +"quantileExactExclusiveState" +"quantileExactForEach" "quantileExactHigh" +"quantileExactHighArgMax" +"quantileExactHighArgMin" +"quantileExactHighArray" +"quantileExactHighDistinct" +"quantileExactHighForEach" +"quantileExactHighIf" +"quantileExactHighMap" +"quantileExactHighMerge" +"quantileExactHighNull" +"quantileExactHighOrDefault" +"quantileExactHighOrNull" +"quantileExactHighResample" +"quantileExactHighSimpleState" +"quantileExactHighState" +"quantileExactIf" "quantileExactInclusive" +"quantileExactInclusiveArgMax" +"quantileExactInclusiveArgMin" +"quantileExactInclusiveArray" +"quantileExactInclusiveDistinct" +"quantileExactInclusiveForEach" +"quantileExactInclusiveIf" +"quantileExactInclusiveMap" +"quantileExactInclusiveMerge" +"quantileExactInclusiveNull" +"quantileExactInclusiveOrDefault" +"quantileExactInclusiveOrNull" +"quantileExactInclusiveResample" +"quantileExactInclusiveSimpleState" +"quantileExactInclusiveState" "quantileExactLow" +"quantileExactLowArgMax" +"quantileExactLowArgMin" +"quantileExactLowArray" +"quantileExactLowDistinct" +"quantileExactLowForEach" +"quantileExactLowIf" +"quantileExactLowMap" +"quantileExactLowMerge" +"quantileExactLowNull" +"quantileExactLowOrDefault" +"quantileExactLowOrNull" +"quantileExactLowResample" +"quantileExactLowSimpleState" +"quantileExactLowState" +"quantileExactMap" +"quantileExactMerge" +"quantileExactNull" +"quantileExactOrDefault" +"quantileExactOrNull" +"quantileExactResample" +"quantileExactSimpleState" +"quantileExactState" "quantileExactWeighted" +"quantileExactWeightedArgMax" +"quantileExactWeightedArgMin" +"quantileExactWeightedArray" +"quantileExactWeightedDistinct" +"quantileExactWeightedForEach" +"quantileExactWeightedIf" +"quantileExactWeightedMap" +"quantileExactWeightedMerge" +"quantileExactWeightedNull" +"quantileExactWeightedOrDefault" +"quantileExactWeightedOrNull" +"quantileExactWeightedResample" +"quantileExactWeightedSimpleState" +"quantileExactWeightedState" +"quantileForEach" +"quantileGK" +"quantileGKArgMax" +"quantileGKArgMin" +"quantileGKArray" +"quantileGKDistinct" +"quantileGKForEach" +"quantileGKIf" +"quantileGKMap" +"quantileGKMerge" +"quantileGKNull" +"quantileGKOrDefault" +"quantileGKOrNull" +"quantileGKResample" +"quantileGKSimpleState" +"quantileGKState" +"quantileIf" +"quantileInterpolatedWeighted" +"quantileInterpolatedWeightedArgMax" +"quantileInterpolatedWeightedArgMin" +"quantileInterpolatedWeightedArray" +"quantileInterpolatedWeightedDistinct" +"quantileInterpolatedWeightedForEach" +"quantileInterpolatedWeightedIf" +"quantileInterpolatedWeightedMap" +"quantileInterpolatedWeightedMerge" +"quantileInterpolatedWeightedNull" +"quantileInterpolatedWeightedOrDefault" +"quantileInterpolatedWeightedOrNull" +"quantileInterpolatedWeightedResample" +"quantileInterpolatedWeightedSimpleState" +"quantileInterpolatedWeightedState" +"quantileMap" +"quantileMerge" +"quantileNull" +"quantileOrDefault" +"quantileOrNull" +"quantileResample" "quantiles" +"quantilesArgMax" +"quantilesArgMin" +"quantilesArray" "quantilesBFloat16" +"quantilesBFloat16ArgMax" +"quantilesBFloat16ArgMin" +"quantilesBFloat16Array" +"quantilesBFloat16Distinct" +"quantilesBFloat16ForEach" +"quantilesBFloat16If" +"quantilesBFloat16Map" +"quantilesBFloat16Merge" +"quantilesBFloat16Null" +"quantilesBFloat16OrDefault" +"quantilesBFloat16OrNull" +"quantilesBFloat16Resample" +"quantilesBFloat16SimpleState" +"quantilesBFloat16State" "quantilesBFloat16Weighted" +"quantilesBFloat16WeightedArgMax" +"quantilesBFloat16WeightedArgMin" +"quantilesBFloat16WeightedArray" +"quantilesBFloat16WeightedDistinct" +"quantilesBFloat16WeightedForEach" +"quantilesBFloat16WeightedIf" +"quantilesBFloat16WeightedMap" +"quantilesBFloat16WeightedMerge" +"quantilesBFloat16WeightedNull" +"quantilesBFloat16WeightedOrDefault" +"quantilesBFloat16WeightedOrNull" +"quantilesBFloat16WeightedResample" +"quantilesBFloat16WeightedSimpleState" +"quantilesBFloat16WeightedState" +"quantilesDD" +"quantilesDDArgMax" +"quantilesDDArgMin" +"quantilesDDArray" +"quantilesDDDistinct" +"quantilesDDForEach" +"quantilesDDIf" +"quantilesDDMap" +"quantilesDDMerge" +"quantilesDDNull" +"quantilesDDOrDefault" +"quantilesDDOrNull" +"quantilesDDResample" +"quantilesDDSimpleState" +"quantilesDDState" "quantilesDeterministic" +"quantilesDeterministicArgMax" +"quantilesDeterministicArgMin" +"quantilesDeterministicArray" +"quantilesDeterministicDistinct" +"quantilesDeterministicForEach" +"quantilesDeterministicIf" +"quantilesDeterministicMap" +"quantilesDeterministicMerge" +"quantilesDeterministicNull" +"quantilesDeterministicOrDefault" +"quantilesDeterministicOrNull" +"quantilesDeterministicResample" +"quantilesDeterministicSimpleState" +"quantilesDeterministicState" +"quantilesDistinct" "quantilesExact" +"quantilesExactArgMax" +"quantilesExactArgMin" +"quantilesExactArray" +"quantilesExactDistinct" "quantilesExactExclusive" +"quantilesExactExclusiveArgMax" +"quantilesExactExclusiveArgMin" +"quantilesExactExclusiveArray" +"quantilesExactExclusiveDistinct" +"quantilesExactExclusiveForEach" +"quantilesExactExclusiveIf" +"quantilesExactExclusiveMap" +"quantilesExactExclusiveMerge" +"quantilesExactExclusiveNull" +"quantilesExactExclusiveOrDefault" +"quantilesExactExclusiveOrNull" +"quantilesExactExclusiveResample" +"quantilesExactExclusiveSimpleState" +"quantilesExactExclusiveState" +"quantilesExactForEach" "quantilesExactHigh" +"quantilesExactHighArgMax" +"quantilesExactHighArgMin" +"quantilesExactHighArray" +"quantilesExactHighDistinct" +"quantilesExactHighForEach" +"quantilesExactHighIf" +"quantilesExactHighMap" +"quantilesExactHighMerge" +"quantilesExactHighNull" +"quantilesExactHighOrDefault" +"quantilesExactHighOrNull" +"quantilesExactHighResample" +"quantilesExactHighSimpleState" +"quantilesExactHighState" +"quantilesExactIf" "quantilesExactInclusive" +"quantilesExactInclusiveArgMax" +"quantilesExactInclusiveArgMin" +"quantilesExactInclusiveArray" +"quantilesExactInclusiveDistinct" +"quantilesExactInclusiveForEach" +"quantilesExactInclusiveIf" +"quantilesExactInclusiveMap" +"quantilesExactInclusiveMerge" +"quantilesExactInclusiveNull" +"quantilesExactInclusiveOrDefault" +"quantilesExactInclusiveOrNull" +"quantilesExactInclusiveResample" +"quantilesExactInclusiveSimpleState" +"quantilesExactInclusiveState" "quantilesExactLow" +"quantilesExactLowArgMax" +"quantilesExactLowArgMin" +"quantilesExactLowArray" +"quantilesExactLowDistinct" +"quantilesExactLowForEach" +"quantilesExactLowIf" +"quantilesExactLowMap" +"quantilesExactLowMerge" +"quantilesExactLowNull" +"quantilesExactLowOrDefault" +"quantilesExactLowOrNull" +"quantilesExactLowResample" +"quantilesExactLowSimpleState" +"quantilesExactLowState" +"quantilesExactMap" +"quantilesExactMerge" +"quantilesExactNull" +"quantilesExactOrDefault" +"quantilesExactOrNull" +"quantilesExactResample" +"quantilesExactSimpleState" +"quantilesExactState" "quantilesExactWeighted" +"quantilesExactWeightedArgMax" +"quantilesExactWeightedArgMin" +"quantilesExactWeightedArray" +"quantilesExactWeightedDistinct" +"quantilesExactWeightedForEach" +"quantilesExactWeightedIf" +"quantilesExactWeightedMap" +"quantilesExactWeightedMerge" +"quantilesExactWeightedNull" +"quantilesExactWeightedOrDefault" +"quantilesExactWeightedOrNull" +"quantilesExactWeightedResample" +"quantilesExactWeightedSimpleState" +"quantilesExactWeightedState" +"quantilesForEach" +"quantilesGK" +"quantilesGKArgMax" +"quantilesGKArgMin" +"quantilesGKArray" +"quantilesGKDistinct" +"quantilesGKForEach" +"quantilesGKIf" +"quantilesGKMap" +"quantilesGKMerge" +"quantilesGKNull" +"quantilesGKOrDefault" +"quantilesGKOrNull" +"quantilesGKResample" +"quantilesGKSimpleState" +"quantilesGKState" +"quantilesIf" +"quantileSimpleState" +"quantilesInterpolatedWeighted" +"quantilesInterpolatedWeightedArgMax" +"quantilesInterpolatedWeightedArgMin" +"quantilesInterpolatedWeightedArray" +"quantilesInterpolatedWeightedDistinct" +"quantilesInterpolatedWeightedForEach" +"quantilesInterpolatedWeightedIf" +"quantilesInterpolatedWeightedMap" +"quantilesInterpolatedWeightedMerge" +"quantilesInterpolatedWeightedNull" +"quantilesInterpolatedWeightedOrDefault" +"quantilesInterpolatedWeightedOrNull" +"quantilesInterpolatedWeightedResample" +"quantilesInterpolatedWeightedSimpleState" +"quantilesInterpolatedWeightedState" +"quantilesMap" +"quantilesMerge" +"quantilesNull" +"quantilesOrDefault" +"quantilesOrNull" +"quantilesResample" +"quantilesSimpleState" +"quantilesState" +"quantileState" "quantilesTDigest" +"quantilesTDigestArgMax" +"quantilesTDigestArgMin" +"quantilesTDigestArray" +"quantilesTDigestDistinct" +"quantilesTDigestForEach" +"quantilesTDigestIf" +"quantilesTDigestMap" +"quantilesTDigestMerge" +"quantilesTDigestNull" +"quantilesTDigestOrDefault" +"quantilesTDigestOrNull" +"quantilesTDigestResample" +"quantilesTDigestSimpleState" +"quantilesTDigestState" "quantilesTDigestWeighted" +"quantilesTDigestWeightedArgMax" +"quantilesTDigestWeightedArgMin" +"quantilesTDigestWeightedArray" +"quantilesTDigestWeightedDistinct" +"quantilesTDigestWeightedForEach" +"quantilesTDigestWeightedIf" +"quantilesTDigestWeightedMap" +"quantilesTDigestWeightedMerge" +"quantilesTDigestWeightedNull" +"quantilesTDigestWeightedOrDefault" +"quantilesTDigestWeightedOrNull" +"quantilesTDigestWeightedResample" +"quantilesTDigestWeightedSimpleState" +"quantilesTDigestWeightedState" "quantilesTiming" +"quantilesTimingArgMax" +"quantilesTimingArgMin" +"quantilesTimingArray" +"quantilesTimingDistinct" +"quantilesTimingForEach" +"quantilesTimingIf" +"quantilesTimingMap" +"quantilesTimingMerge" +"quantilesTimingNull" +"quantilesTimingOrDefault" +"quantilesTimingOrNull" +"quantilesTimingResample" +"quantilesTimingSimpleState" +"quantilesTimingState" "quantilesTimingWeighted" +"quantilesTimingWeightedArgMax" +"quantilesTimingWeightedArgMin" +"quantilesTimingWeightedArray" +"quantilesTimingWeightedDistinct" +"quantilesTimingWeightedForEach" +"quantilesTimingWeightedIf" +"quantilesTimingWeightedMap" +"quantilesTimingWeightedMerge" +"quantilesTimingWeightedNull" +"quantilesTimingWeightedOrDefault" +"quantilesTimingWeightedOrNull" +"quantilesTimingWeightedResample" +"quantilesTimingWeightedSimpleState" +"quantilesTimingWeightedState" "quantileTDigest" +"quantileTDigestArgMax" +"quantileTDigestArgMin" +"quantileTDigestArray" +"quantileTDigestDistinct" +"quantileTDigestForEach" +"quantileTDigestIf" +"quantileTDigestMap" +"quantileTDigestMerge" +"quantileTDigestNull" +"quantileTDigestOrDefault" +"quantileTDigestOrNull" +"quantileTDigestResample" +"quantileTDigestSimpleState" +"quantileTDigestState" "quantileTDigestWeighted" +"quantileTDigestWeightedArgMax" +"quantileTDigestWeightedArgMin" +"quantileTDigestWeightedArray" +"quantileTDigestWeightedDistinct" +"quantileTDigestWeightedForEach" +"quantileTDigestWeightedIf" +"quantileTDigestWeightedMap" +"quantileTDigestWeightedMerge" +"quantileTDigestWeightedNull" +"quantileTDigestWeightedOrDefault" +"quantileTDigestWeightedOrNull" +"quantileTDigestWeightedResample" +"quantileTDigestWeightedSimpleState" +"quantileTDigestWeightedState" "quantileTiming" +"quantileTimingArgMax" +"quantileTimingArgMin" +"quantileTimingArray" +"quantileTimingDistinct" +"quantileTimingForEach" +"quantileTimingIf" +"quantileTimingMap" +"quantileTimingMerge" +"quantileTimingNull" +"quantileTimingOrDefault" +"quantileTimingOrNull" +"quantileTimingResample" +"quantileTimingSimpleState" +"quantileTimingState" "quantileTimingWeighted" +"quantileTimingWeightedArgMax" +"quantileTimingWeightedArgMin" +"quantileTimingWeightedArray" +"quantileTimingWeightedDistinct" +"quantileTimingWeightedForEach" +"quantileTimingWeightedIf" +"quantileTimingWeightedMap" +"quantileTimingWeightedMerge" +"quantileTimingWeightedNull" +"quantileTimingWeightedOrDefault" +"quantileTimingWeightedOrNull" +"quantileTimingWeightedResample" +"quantileTimingWeightedSimpleState" +"quantileTimingWeightedState" "QUARTER" +"QUARTERS" +"QUERY" "query_id" "queryID" "queryString" "queryStringAndFragment" +"QUERY TREE" +"QUOTA" +"radians" "rand" "rand32" "rand64" +"randBernoulli" +"randBinomial" +"randCanonical" +"randChiSquared" "randConstant" +"randExponential" +"randFisherF" +"randLogNormal" +"randNegativeBinomial" +"randNormal" "randomFixedString" +"RANDOMIZED" +"RANDOMIZE FOR" "randomPrintableASCII" "randomString" "randomStringUTF8" +"randPoisson" +"randStudentT" +"randUniform" "range" "RANGE" "rank" +"rankArgMax" +"rankArgMin" +"rankArray" "rankCorr" +"rankCorrArgMax" +"rankCorrArgMin" +"rankCorrArray" +"rankCorrDistinct" +"rankCorrForEach" +"rankCorrIf" +"rankCorrMap" +"rankCorrMerge" +"rankCorrNull" +"rankCorrOrDefault" +"rankCorrOrNull" +"rankCorrResample" +"rankCorrSimpleState" +"rankCorrState" +"rankDistinct" +"rankForEach" +"rankIf" +"rankMap" +"rankMerge" +"rankNull" +"rankOrDefault" +"rankOrNull" +"rankResample" +"rankSimpleState" +"rankState" +"READONLY" "readWKTMultiPolygon" "readWKTPoint" "readWKTPolygon" "readWKTRing" "REAL" +"REALM" +"RECOMPRESS" +"REFERENCES" "REFRESH" +"REGEXP" +"regexpExtract" +"REGEXP_EXTRACT" +"REGEXP_MATCHES" "regexpQuoteMeta" +"REGEXP_REPLACE" "regionHierarchy" "regionIn" "regionToArea" @@ -1030,11 +3526,15 @@ "reinterpretAsUInt64" "reinterpretAsUInt8" "reinterpretAsUUID" -"RELOAD" "REMOVE" +"REMOVE SAMPLE BY" +"REMOVE TTL" "RENAME" "RENAME COLUMN" +"RENAME DATABASE" +"RENAME DICTIONARY" "RENAME TABLE" +"RENAME TO" "repeat" "replace" "REPLACE" @@ -1043,18 +3543,40 @@ "REPLACE PARTITION" "replaceRegexpAll" "replaceRegexpOne" -"REPLICA" "replicate" -"REPLICATED" "Resample" +"RESET SETTING" +"RESPECT NULLS" +"RESTORE" +"RESTRICT" +"RESTRICTIVE" "RESUME" "retention" +"retentionArgMax" +"retentionArgMin" +"retentionArray" +"retentionDistinct" +"retentionForEach" +"retentionIf" +"retentionMap" +"retentionMerge" +"retentionNull" +"retentionOrDefault" +"retentionOrNull" +"retentionResample" +"retentionSimpleState" +"retentionState" "reverse" "reverseUTF8" +"revision" +"REVOKE" +"right" "RIGHT" "rightPad" "rightPadUTF8" +"rightUTF8" "Ring" +"ROLLBACK" "ROLLUP" "round" "roundAge" @@ -1062,10 +3584,27 @@ "roundDown" "roundDuration" "roundToExp2" +"ROW" "row_number" +"row_numberArgMax" +"row_numberArgMin" +"row_numberArray" +"row_numberDistinct" +"row_numberForEach" +"row_numberIf" "rowNumberInAllBlocks" "rowNumberInBlock" +"row_numberMap" +"row_numberMerge" +"row_numberNull" +"row_numberOrDefault" +"row_numberOrNull" +"row_numberResample" +"row_numberSimpleState" +"row_numberState" +"ROWS" "rpad" +"rtrim" "RTRIM" "runningAccumulate" "runningConcurrency" @@ -1081,29 +3620,103 @@ "s2RectIntersection" "s2RectUnion" "s2ToGeo" +"S3" +"SALT" "SAMPLE" "SAMPLE BY" +"scalarProduct" +"__scalarSubqueryResult" +"SCHEMA" +"SCHEME" "SECOND" +"SECONDS" "SELECT" "SEMI" -"SENDS" "sequenceCount" +"sequenceCountArgMax" +"sequenceCountArgMin" +"sequenceCountArray" +"sequenceCountDistinct" +"sequenceCountForEach" +"sequenceCountIf" +"sequenceCountMap" +"sequenceCountMerge" +"sequenceCountNull" +"sequenceCountOrDefault" +"sequenceCountOrNull" +"sequenceCountResample" +"sequenceCountSimpleState" +"sequenceCountState" "sequenceMatch" +"sequenceMatchArgMax" +"sequenceMatchArgMin" +"sequenceMatchArray" +"sequenceMatchDistinct" +"sequenceMatchForEach" +"sequenceMatchIf" +"sequenceMatchMap" +"sequenceMatchMerge" +"sequenceMatchNull" +"sequenceMatchOrDefault" +"sequenceMatchOrNull" +"sequenceMatchResample" +"sequenceMatchSimpleState" +"sequenceMatchState" "sequenceNextNode" +"sequenceNextNodeArgMax" +"sequenceNextNodeArgMin" +"sequenceNextNodeArray" +"sequenceNextNodeDistinct" +"sequenceNextNodeForEach" +"sequenceNextNodeIf" +"sequenceNextNodeMap" +"sequenceNextNodeMerge" +"sequenceNextNodeNull" +"sequenceNextNodeOrDefault" +"sequenceNextNodeOrNull" +"sequenceNextNodeResample" +"sequenceNextNodeSimpleState" +"sequenceNextNodeState" +"seriesDecomposeSTL" +"seriesOutliersDetectTukey" +"seriesPeriodDetectFFT" +"SERVER" +"serverTimezone" +"serverTimeZone" "serverUUID" "SET" +"SET DEFAULT" +"SET DEFAULT ROLE" +"SET FAKE TIME" +"SET NULL" +"SET ROLE" +"SET ROLE DEFAULT" "SETTINGS" +"SET TRANSACTION SNAPSHOT" "SHA1" "SHA224" "SHA256" +"SHA256_HASH" +"SHA256_PASSWORD" "SHA384" "SHA512" +"SHA512_256" "shardCount" "shardNum" "SHOW" +"SHOW ACCESS" +"showCertificate" +"SHOW CREATE" +"SHOW ENGINES" +"SHOW FUNCTIONS" +"SHOW GRANTS" +"SHOW PRIVILEGES" "SHOW PROCESSLIST" +"SHOW SETTING" "sigmoid" "sign" +"SIGNED" +"SIMPLE" "SimpleAggregateFunction" "simpleJSONExtractBool" "simpleJSONExtractFloat" @@ -1113,14 +3726,74 @@ "simpleJSONExtractUInt" "simpleJSONHas" "simpleLinearRegression" +"simpleLinearRegressionArgMax" +"simpleLinearRegressionArgMin" +"simpleLinearRegressionArray" +"simpleLinearRegressionDistinct" +"simpleLinearRegressionForEach" +"simpleLinearRegressionIf" +"simpleLinearRegressionMap" +"simpleLinearRegressionMerge" +"simpleLinearRegressionNull" +"simpleLinearRegressionOrDefault" +"simpleLinearRegressionOrNull" +"simpleLinearRegressionResample" +"simpleLinearRegressionSimpleState" +"simpleLinearRegressionState" "sin" "SINGLE" "singleValueOrNull" +"singleValueOrNullArgMax" +"singleValueOrNullArgMin" +"singleValueOrNullArray" +"singleValueOrNullDistinct" +"singleValueOrNullForEach" +"singleValueOrNullIf" +"singleValueOrNullMap" +"singleValueOrNullMerge" +"singleValueOrNullNull" +"singleValueOrNullOrDefault" +"singleValueOrNullOrNull" +"singleValueOrNullResample" +"singleValueOrNullSimpleState" +"singleValueOrNullState" "sinh" "sipHash128" +"sipHash128Keyed" +"sipHash128Reference" +"sipHash128ReferenceKeyed" "sipHash64" +"sipHash64Keyed" "skewPop" +"skewPopArgMax" +"skewPopArgMin" +"skewPopArray" +"skewPopDistinct" +"skewPopForEach" +"skewPopIf" +"skewPopMap" +"skewPopMerge" +"skewPopNull" +"skewPopOrDefault" +"skewPopOrNull" +"skewPopResample" +"skewPopSimpleState" +"skewPopState" "skewSamp" +"skewSampArgMax" +"skewSampArgMin" +"skewSampArray" +"skewSampDistinct" +"skewSampForEach" +"skewSampIf" +"skewSampMap" +"skewSampMerge" +"skewSampNull" +"skewSampOrDefault" +"skewSampOrNull" +"skewSampResample" +"skewSampSimpleState" +"skewSampState" "sleep" "sleepEachRow" "SMALLINT" @@ -1128,61 +3801,377 @@ "SMALLINT UNSIGNED" "snowflakeToDateTime" "snowflakeToDateTime64" +"soundex" "SOURCE" +"space" "sparkbar" +"sparkBar" +"sparkbarArgMax" +"sparkBarArgMax" +"sparkbarArgMin" +"sparkBarArgMin" +"sparkbarArray" +"sparkBarArray" +"sparkbarDistinct" +"sparkBarDistinct" +"sparkbarForEach" +"sparkBarForEach" +"sparkbarIf" +"sparkBarIf" +"sparkbarMap" +"sparkBarMap" +"sparkbarMerge" +"sparkBarMerge" +"sparkbarNull" +"sparkBarNull" +"sparkbarOrDefault" +"sparkBarOrDefault" +"sparkbarOrNull" +"sparkBarOrNull" +"sparkbarResample" +"sparkBarResample" +"sparkbarSimpleState" +"sparkBarSimpleState" +"sparkbarState" +"sparkBarState" +"SPATIAL" +"splitByAlpha" "splitByChar" "splitByNonAlpha" "splitByRegexp" "splitByString" "splitByWhitespace" +"sqid" +"sqidDecode" +"sqidEncode" +"SQL SECURITY" "SQL_TSI_DAY" "SQL_TSI_HOUR" +"SQL_TSI_MICROSECOND" +"SQL_TSI_MILLISECOND" "SQL_TSI_MINUTE" "SQL_TSI_MONTH" +"SQL_TSI_NANOSECOND" "SQL_TSI_QUARTER" "SQL_TSI_SECOND" "SQL_TSI_WEEK" "SQL_TSI_YEAR" "sqrt" "SS" -"START" +"SSH_KEY" +"SSL_CERTIFICATE" "startsWith" +"startsWithUTF8" "State" +"STATISTIC" +"STD" +"STDArgMax" +"STDArgMin" +"STDArray" "stddevPop" "STDDEV_POP" +"stddevPopArgMax" +"STDDEV_POPArgMax" +"stddevPopArgMin" +"STDDEV_POPArgMin" +"stddevPopArray" +"STDDEV_POPArray" +"stddevPopDistinct" +"STDDEV_POPDistinct" +"stddevPopForEach" +"STDDEV_POPForEach" +"stddevPopIf" +"STDDEV_POPIf" +"stddevPopMap" +"STDDEV_POPMap" +"stddevPopMerge" +"STDDEV_POPMerge" +"stddevPopNull" +"STDDEV_POPNull" +"stddevPopOrDefault" +"STDDEV_POPOrDefault" +"stddevPopOrNull" +"STDDEV_POPOrNull" +"stddevPopResample" +"STDDEV_POPResample" +"stddevPopSimpleState" +"STDDEV_POPSimpleState" "stddevPopStable" +"stddevPopStableArgMax" +"stddevPopStableArgMin" +"stddevPopStableArray" +"stddevPopStableDistinct" +"stddevPopStableForEach" +"stddevPopStableIf" +"stddevPopStableMap" +"stddevPopStableMerge" +"stddevPopStableNull" +"stddevPopStableOrDefault" +"stddevPopStableOrNull" +"stddevPopStableResample" +"stddevPopStableSimpleState" +"stddevPopStableState" +"stddevPopState" +"STDDEV_POPState" "stddevSamp" "STDDEV_SAMP" +"stddevSampArgMax" +"STDDEV_SAMPArgMax" +"stddevSampArgMin" +"STDDEV_SAMPArgMin" +"stddevSampArray" +"STDDEV_SAMPArray" +"stddevSampDistinct" +"STDDEV_SAMPDistinct" +"stddevSampForEach" +"STDDEV_SAMPForEach" +"stddevSampIf" +"STDDEV_SAMPIf" +"stddevSampMap" +"STDDEV_SAMPMap" +"stddevSampMerge" +"STDDEV_SAMPMerge" +"stddevSampNull" +"STDDEV_SAMPNull" +"stddevSampOrDefault" +"STDDEV_SAMPOrDefault" +"stddevSampOrNull" +"STDDEV_SAMPOrNull" +"stddevSampResample" +"STDDEV_SAMPResample" +"stddevSampSimpleState" +"STDDEV_SAMPSimpleState" "stddevSampStable" +"stddevSampStableArgMax" +"stddevSampStableArgMin" +"stddevSampStableArray" +"stddevSampStableDistinct" +"stddevSampStableForEach" +"stddevSampStableIf" +"stddevSampStableMap" +"stddevSampStableMerge" +"stddevSampStableNull" +"stddevSampStableOrDefault" +"stddevSampStableOrNull" +"stddevSampStableResample" +"stddevSampStableSimpleState" +"stddevSampStableState" +"stddevSampState" +"STDDEV_SAMPState" +"STDDistinct" +"STDForEach" +"STDIf" +"STDMap" +"STDMerge" +"STDNull" +"STDOrDefault" +"STDOrNull" +"STDResample" +"STDSimpleState" +"STDState" "stem" "STEP" "stochasticLinearRegression" +"stochasticLinearRegressionArgMax" +"stochasticLinearRegressionArgMin" +"stochasticLinearRegressionArray" +"stochasticLinearRegressionDistinct" +"stochasticLinearRegressionForEach" +"stochasticLinearRegressionIf" +"stochasticLinearRegressionMap" +"stochasticLinearRegressionMerge" +"stochasticLinearRegressionNull" +"stochasticLinearRegressionOrDefault" +"stochasticLinearRegressionOrNull" +"stochasticLinearRegressionResample" +"stochasticLinearRegressionSimpleState" +"stochasticLinearRegressionState" "stochasticLogisticRegression" -"STOP" +"stochasticLogisticRegressionArgMax" +"stochasticLogisticRegressionArgMin" +"stochasticLogisticRegressionArray" +"stochasticLogisticRegressionDistinct" +"stochasticLogisticRegressionForEach" +"stochasticLogisticRegressionIf" +"stochasticLogisticRegressionMap" +"stochasticLogisticRegressionMerge" +"stochasticLogisticRegressionNull" +"stochasticLogisticRegressionOrDefault" +"stochasticLogisticRegressionOrNull" +"stochasticLogisticRegressionResample" +"stochasticLogisticRegressionSimpleState" +"stochasticLogisticRegressionState" +"STORAGE" +"STRICT" +"STRICTLY_ASCENDING" "String" +"stringJaccardIndex" +"stringJaccardIndexUTF8" "stringToH3" +"str_to_date" +"str_to_map" +"structureToCapnProtoSchema" +"structureToProtobufSchema" "studentTTest" +"studentTTestArgMax" +"studentTTestArgMin" +"studentTTestArray" +"studentTTestDistinct" +"studentTTestForEach" +"studentTTestIf" +"studentTTestMap" +"studentTTestMerge" +"studentTTestNull" +"studentTTestOrDefault" +"studentTTestOrNull" +"studentTTestResample" +"studentTTestSimpleState" +"studentTTestState" "subBitmap" +"subDate" +"SUBPARTITION" +"SUBPARTITION BY" +"SUBPARTITIONS" "substr" "substring" "SUBSTRING" +"substringIndex" +"SUBSTRING_INDEX" +"substringIndexUTF8" "substringUTF8" "subtractDays" "subtractHours" +"subtractInterval" +"subtractMicroseconds" +"subtractMilliseconds" "subtractMinutes" "subtractMonths" +"subtractNanoseconds" "subtractQuarters" "subtractSeconds" +"subtractTupleOfIntervals" "subtractWeeks" "subtractYears" "sum" +"sumArgMax" +"sumArgMin" +"sumArray" "sumCount" +"sumCountArgMax" +"sumCountArgMin" +"sumCountArray" +"sumCountDistinct" +"sumCountForEach" +"sumCountIf" +"sumCountMap" +"sumCountMerge" +"sumCountNull" +"sumCountOrDefault" +"sumCountOrNull" +"sumCountResample" +"sumCountSimpleState" +"sumCountState" +"sumDistinct" +"sumForEach" +"sumIf" "sumKahan" +"sumKahanArgMax" +"sumKahanArgMin" +"sumKahanArray" +"sumKahanDistinct" +"sumKahanForEach" +"sumKahanIf" +"sumKahanMap" +"sumKahanMerge" +"sumKahanNull" +"sumKahanOrDefault" +"sumKahanOrNull" +"sumKahanResample" +"sumKahanSimpleState" +"sumKahanState" "sumMap" "sumMapFiltered" +"sumMapFilteredArgMax" +"sumMapFilteredArgMin" +"sumMapFilteredArray" +"sumMapFilteredDistinct" +"sumMapFilteredForEach" +"sumMapFilteredIf" +"sumMapFilteredMap" +"sumMapFilteredMerge" +"sumMapFilteredNull" +"sumMapFilteredOrDefault" +"sumMapFilteredOrNull" +"sumMapFilteredResample" +"sumMapFilteredSimpleState" +"sumMapFilteredState" "sumMapFilteredWithOverflow" +"sumMapFilteredWithOverflowArgMax" +"sumMapFilteredWithOverflowArgMin" +"sumMapFilteredWithOverflowArray" +"sumMapFilteredWithOverflowDistinct" +"sumMapFilteredWithOverflowForEach" +"sumMapFilteredWithOverflowIf" +"sumMapFilteredWithOverflowMap" +"sumMapFilteredWithOverflowMerge" +"sumMapFilteredWithOverflowNull" +"sumMapFilteredWithOverflowOrDefault" +"sumMapFilteredWithOverflowOrNull" +"sumMapFilteredWithOverflowResample" +"sumMapFilteredWithOverflowSimpleState" +"sumMapFilteredWithOverflowState" +"sumMappedArrays" +"sumMappedArraysArgMax" +"sumMappedArraysArgMin" +"sumMappedArraysArray" +"sumMappedArraysDistinct" +"sumMappedArraysForEach" +"sumMappedArraysIf" +"sumMappedArraysMap" +"sumMappedArraysMerge" +"sumMappedArraysNull" +"sumMappedArraysOrDefault" +"sumMappedArraysOrNull" +"sumMappedArraysResample" +"sumMappedArraysSimpleState" +"sumMappedArraysState" "sumMapWithOverflow" +"sumMapWithOverflowArgMax" +"sumMapWithOverflowArgMin" +"sumMapWithOverflowArray" +"sumMapWithOverflowDistinct" +"sumMapWithOverflowForEach" +"sumMapWithOverflowIf" +"sumMapWithOverflowMap" +"sumMapWithOverflowMerge" +"sumMapWithOverflowNull" +"sumMapWithOverflowOrDefault" +"sumMapWithOverflowOrNull" +"sumMapWithOverflowResample" +"sumMapWithOverflowSimpleState" +"sumMapWithOverflowState" +"sumMerge" +"sumNull" +"sumOrDefault" +"sumOrNull" +"sumResample" +"sumSimpleState" +"sumState" "sumWithOverflow" +"sumWithOverflowArgMax" +"sumWithOverflowArgMin" +"sumWithOverflowArray" +"sumWithOverflowDistinct" +"sumWithOverflowForEach" +"sumWithOverflowIf" +"sumWithOverflowMap" +"sumWithOverflowMerge" +"sumWithOverflowNull" +"sumWithOverflowOrDefault" +"sumWithOverflowOrNull" +"sumWithOverflowResample" +"sumWithOverflowSimpleState" +"sumWithOverflowState" "SUSPEND" "svg" "SVG" @@ -1191,24 +4180,44 @@ "SYNTAX" "SYSTEM" "TABLE" +"TABLE OVERRIDE" "TABLES" "tan" "tanh" "tcpPort" "TEMPORARY" +"TEMPORARY TABLE" "TEST" "TEXT" "tgamma" +"theilsU" +"theilsUArgMax" +"theilsUArgMin" +"theilsUArray" +"theilsUDistinct" +"theilsUForEach" +"theilsUIf" +"theilsUMap" +"theilsUMerge" +"theilsUNull" +"theilsUOrDefault" +"theilsUOrNull" +"theilsUResample" +"theilsUSimpleState" +"theilsUState" "THEN" "throwIf" "tid" -"TIES" -"TIMEOUT" +"TIME" +"timeDiff" "timeSlot" "timeSlots" +"timestamp" "TIMESTAMP" "TIMESTAMP_ADD" "TIMESTAMPADD" +"timestamp_diff" +"timestampDiff" "TIMESTAMP_DIFF" "TIMESTAMPDIFF" "TIMESTAMP_SUB" @@ -1226,77 +4235,111 @@ "TINYTEXT" "TO" "TO_BASE64" +"toBool" "toColumnTypeName" "toDate" "toDate32" +"toDate32OrDefault" "toDate32OrNull" "toDate32OrZero" +"toDateOrDefault" "toDateOrNull" "toDateOrZero" "toDateTime" "toDateTime32" "toDateTime64" +"toDateTime64OrDefault" "toDateTime64OrNull" "toDateTime64OrZero" +"toDateTimeOrDefault" "toDateTimeOrNull" "toDateTimeOrZero" "today" "toDayOfMonth" "toDayOfWeek" "toDayOfYear" +"TO_DAYS" +"toDaysSinceYearZero" "toDecimal128" +"toDecimal128OrDefault" "toDecimal128OrNull" "toDecimal128OrZero" "toDecimal256" +"toDecimal256OrDefault" "toDecimal256OrNull" "toDecimal256OrZero" "toDecimal32" +"toDecimal32OrDefault" "toDecimal32OrNull" "toDecimal32OrZero" "toDecimal64" +"toDecimal64OrDefault" "toDecimal64OrNull" "toDecimal64OrZero" +"toDecimalString" "TO DISK" "toFixedString" "toFloat32" +"toFloat32OrDefault" "toFloat32OrNull" "toFloat32OrZero" "toFloat64" +"toFloat64OrDefault" "toFloat64OrNull" "toFloat64OrZero" "toHour" +"TO INNER UUID" "toInt128" +"toInt128OrDefault" "toInt128OrNull" "toInt128OrZero" "toInt16" +"toInt16OrDefault" "toInt16OrNull" "toInt16OrZero" "toInt256" +"toInt256OrDefault" "toInt256OrNull" "toInt256OrZero" "toInt32" +"toInt32OrDefault" "toInt32OrNull" "toInt32OrZero" "toInt64" +"toInt64OrDefault" "toInt64OrNull" "toInt64OrZero" "toInt8" +"toInt8OrDefault" "toInt8OrNull" "toInt8OrZero" "toIntervalDay" "toIntervalHour" +"toIntervalMicrosecond" +"toIntervalMillisecond" "toIntervalMinute" "toIntervalMonth" +"toIntervalNanosecond" "toIntervalQuarter" "toIntervalSecond" "toIntervalWeek" "toIntervalYear" "toIPv4" +"toIPv4OrDefault" +"toIPv4OrNull" +"toIPv4OrZero" "toIPv6" +"toIPv6OrDefault" +"toIPv6OrNull" +"toIPv6OrZero" "toISOWeek" "toISOYear" "toJSONString" +"tokens" +"toLastDayOfMonth" +"toLastDayOfWeek" "toLowCardinality" +"toMillisecond" "toMinute" "toModifiedJulianDay" "toModifiedJulianDayOrNull" @@ -1305,8 +4348,37 @@ "toNullable" "TOP" "topK" +"topKArgMax" +"topKArgMin" +"topKArray" +"topKDistinct" +"topKForEach" +"topKIf" +"topKMap" +"topKMerge" +"topKNull" +"topKOrDefault" +"topKOrNull" +"topKResample" +"topKSimpleState" +"topKState" "topKWeighted" +"topKWeightedArgMax" +"topKWeightedArgMin" +"topKWeightedArray" +"topKWeightedDistinct" +"topKWeightedForEach" +"topKWeightedIf" +"topKWeightedMap" +"topKWeightedMerge" +"topKWeightedNull" +"topKWeightedOrDefault" +"topKWeightedOrNull" +"topKWeightedResample" +"topKWeightedSimpleState" +"topKWeightedState" "topLevelDomain" +"topLevelDomainRFC" "toQuarter" "toRelativeDayNum" "toRelativeHourNum" @@ -1317,14 +4389,19 @@ "toRelativeWeekNum" "toRelativeYearNum" "toSecond" +"TO SHARD" "toStartOfDay" "toStartOfFifteenMinutes" +"toStartOfFiveMinute" "toStartOfFiveMinutes" "toStartOfHour" "toStartOfInterval" "toStartOfISOYear" +"toStartOfMicrosecond" +"toStartOfMillisecond" "toStartOfMinute" "toStartOfMonth" +"toStartOfNanosecond" "toStartOfQuarter" "toStartOfSecond" "toStartOfTenMinutes" @@ -1339,28 +4416,38 @@ "toTimeZone" "toTypeName" "toUInt128" +"toUInt128OrDefault" "toUInt128OrNull" "toUInt128OrZero" "toUInt16" +"toUInt16OrDefault" "toUInt16OrNull" "toUInt16OrZero" "toUInt256" +"toUInt256OrDefault" "toUInt256OrNull" "toUInt256OrZero" "toUInt32" +"toUInt32OrDefault" "toUInt32OrNull" "toUInt32OrZero" "toUInt64" +"toUInt64OrDefault" "toUInt64OrNull" "toUInt64OrZero" "toUInt8" +"toUInt8OrDefault" "toUInt8OrNull" "toUInt8OrZero" +"TO_UNIXTIME" "toUnixTimestamp" "toUnixTimestamp64Micro" "toUnixTimestamp64Milli" "toUnixTimestamp64Nano" +"to_utc_timestamp" +"toUTCTimestamp" "toUUID" +"toUUIDOrDefault" "toUUIDOrNull" "toUUIDOrZero" "toValidUTF8" @@ -1371,23 +4458,55 @@ "toYYYYMM" "toYYYYMMDD" "toYYYYMMDDhhmmss" +"TRACKING ONLY" "TRAILING" +"TRANSACTION" +"transactionID" +"transactionLatestSnapshot" +"transactionOldestSnapshot" "transform" +"translate" +"translateUTF8" +"TRIGGER" +"trim" "TRIM" "trimBoth" "trimLeft" "trimRight" +"TRUE" "trunc" "truncate" "TRUNCATE" +"tryBase58Decode" "tryBase64Decode" +"tryDecrypt" +"tryIdnaEncode" +"tryPunycodeDecode" "TTL" +"tumble" +"tumbleEnd" +"tumbleStart" "tuple" "Tuple" +"tupleConcat" +"tupleDivide" +"tupleDivideByNumber" "tupleElement" "tupleHammingDistance" +"tupleIntDiv" +"tupleIntDivByNumber" +"tupleIntDivOrZero" +"tupleIntDivOrZeroByNumber" +"tupleMinus" +"tupleModulo" +"tupleModuloByNumber" +"tupleMultiply" +"tupleMultiplyByNumber" +"tupleNegate" +"tuplePlus" "tupleToNameValuePairs" "TYPE" +"TYPEOF" "ucase" "UInt128" "UInt16" @@ -1395,41 +4514,245 @@ "UInt32" "UInt64" "UInt8" +"ULIDStringToDateTime" "unbin" +"UNBOUNDED" +"UNDROP" +"UNFREEZE" "unhex" "UNION" "uniq" +"uniqArgMax" +"uniqArgMin" +"uniqArray" "uniqCombined" "uniqCombined64" +"uniqCombined64ArgMax" +"uniqCombined64ArgMin" +"uniqCombined64Array" +"uniqCombined64Distinct" +"uniqCombined64ForEach" +"uniqCombined64If" +"uniqCombined64Map" +"uniqCombined64Merge" +"uniqCombined64Null" +"uniqCombined64OrDefault" +"uniqCombined64OrNull" +"uniqCombined64Resample" +"uniqCombined64SimpleState" +"uniqCombined64State" +"uniqCombinedArgMax" +"uniqCombinedArgMin" +"uniqCombinedArray" +"uniqCombinedDistinct" +"uniqCombinedForEach" +"uniqCombinedIf" +"uniqCombinedMap" +"uniqCombinedMerge" +"uniqCombinedNull" +"uniqCombinedOrDefault" +"uniqCombinedOrNull" +"uniqCombinedResample" +"uniqCombinedSimpleState" +"uniqCombinedState" +"uniqDistinct" "uniqExact" +"uniqExactArgMax" +"uniqExactArgMin" +"uniqExactArray" +"uniqExactDistinct" +"uniqExactForEach" +"uniqExactIf" +"uniqExactMap" +"uniqExactMerge" +"uniqExactNull" +"uniqExactOrDefault" +"uniqExactOrNull" +"uniqExactResample" +"uniqExactSimpleState" +"uniqExactState" +"uniqForEach" "uniqHLL12" +"uniqHLL12ArgMax" +"uniqHLL12ArgMin" +"uniqHLL12Array" +"uniqHLL12Distinct" +"uniqHLL12ForEach" +"uniqHLL12If" +"uniqHLL12Map" +"uniqHLL12Merge" +"uniqHLL12Null" +"uniqHLL12OrDefault" +"uniqHLL12OrNull" +"uniqHLL12Resample" +"uniqHLL12SimpleState" +"uniqHLL12State" +"uniqIf" +"uniqMap" +"uniqMerge" +"uniqNull" +"uniqOrDefault" +"uniqOrNull" +"uniqResample" +"uniqSimpleState" +"uniqState" "uniqTheta" +"uniqThetaArgMax" +"uniqThetaArgMin" +"uniqThetaArray" +"uniqThetaDistinct" +"uniqThetaForEach" +"uniqThetaIf" +"uniqThetaIntersect" +"uniqThetaMap" +"uniqThetaMerge" +"uniqThetaNot" +"uniqThetaNull" +"uniqThetaOrDefault" +"uniqThetaOrNull" +"uniqThetaResample" +"uniqThetaSimpleState" +"uniqThetaState" +"uniqThetaUnion" +"UNIQUE" "uniqUpTo" +"uniqUpToArgMax" +"uniqUpToArgMin" +"uniqUpToArray" +"uniqUpToDistinct" +"uniqUpToForEach" +"uniqUpToIf" +"uniqUpToMap" +"uniqUpToMerge" +"uniqUpToNull" +"uniqUpToOrDefault" +"uniqUpToOrNull" +"uniqUpToResample" +"uniqUpToSimpleState" +"uniqUpToState" +"UNSET FAKE TIME" +"UNSIGNED" "UPDATE" "upper" "upperUTF8" "uptime" +"URL" "URLHash" "URLHierarchy" "URLPathHierarchy" "USE" "user" "USING" +"UTC_timestamp" +"UTCTimestamp" "UUID" "UUIDNumToString" "UUIDStringToNum" "validateNestedArraySizes" +"VALID UNTIL" "VALUES" +"VARBINARY" "VARCHAR" "VARCHAR2" +"Variant" +"variantElement" +"variantType" "varPop" "VAR_POP" +"varPopArgMax" +"VAR_POPArgMax" +"varPopArgMin" +"VAR_POPArgMin" +"varPopArray" +"VAR_POPArray" +"varPopDistinct" +"VAR_POPDistinct" +"varPopForEach" +"VAR_POPForEach" +"varPopIf" +"VAR_POPIf" +"varPopMap" +"VAR_POPMap" +"varPopMerge" +"VAR_POPMerge" +"varPopNull" +"VAR_POPNull" +"varPopOrDefault" +"VAR_POPOrDefault" +"varPopOrNull" +"VAR_POPOrNull" +"varPopResample" +"VAR_POPResample" +"varPopSimpleState" +"VAR_POPSimpleState" "varPopStable" +"varPopStableArgMax" +"varPopStableArgMin" +"varPopStableArray" +"varPopStableDistinct" +"varPopStableForEach" +"varPopStableIf" +"varPopStableMap" +"varPopStableMerge" +"varPopStableNull" +"varPopStableOrDefault" +"varPopStableOrNull" +"varPopStableResample" +"varPopStableSimpleState" +"varPopStableState" +"varPopState" +"VAR_POPState" "varSamp" "VAR_SAMP" +"varSampArgMax" +"VAR_SAMPArgMax" +"varSampArgMin" +"VAR_SAMPArgMin" +"varSampArray" +"VAR_SAMPArray" +"varSampDistinct" +"VAR_SAMPDistinct" +"varSampForEach" +"VAR_SAMPForEach" +"varSampIf" +"VAR_SAMPIf" +"varSampMap" +"VAR_SAMPMap" +"varSampMerge" +"VAR_SAMPMerge" +"varSampNull" +"VAR_SAMPNull" +"varSampOrDefault" +"VAR_SAMPOrDefault" +"varSampOrNull" +"VAR_SAMPOrNull" +"varSampResample" +"VAR_SAMPResample" +"varSampSimpleState" +"VAR_SAMPSimpleState" "varSampStable" +"varSampStableArgMax" +"varSampStableArgMin" +"varSampStableArray" +"varSampStableDistinct" +"varSampStableForEach" +"varSampStableIf" +"varSampStableMap" +"varSampStableMerge" +"varSampStableNull" +"varSampStableOrDefault" +"varSampStableOrNull" +"varSampStableResample" +"varSampStableSimpleState" +"varSampStableState" +"varSampState" +"VAR_SAMPState" +"VARYING" +"vectorDifference" +"vectorSum" "version" "VIEW" +"VISIBLE" "visibleWidth" "visitParamExtractBool" "visitParamExtractFloat" @@ -1438,16 +4761,55 @@ "visitParamExtractString" "visitParamExtractUInt" "visitParamHas" -"VOLUME" "WATCH" +"WATERMARK" "week" "WEEK" +"WEEKS" "welchTTest" +"welchTTestArgMax" +"welchTTestArgMin" +"welchTTestArray" +"welchTTestDistinct" +"welchTTestForEach" +"welchTTestIf" +"welchTTestMap" +"welchTTestMerge" +"welchTTestNull" +"welchTTestOrDefault" +"welchTTestOrNull" +"welchTTestResample" +"welchTTestSimpleState" +"welchTTestState" "WHEN" "WHERE" +"width_bucket" +"widthBucket" +"WINDOW" "windowFunnel" +"windowFunnelArgMax" +"windowFunnelArgMin" +"windowFunnelArray" +"windowFunnelDistinct" +"windowFunnelForEach" +"windowFunnelIf" +"windowFunnelMap" +"windowFunnelMerge" +"windowFunnelNull" +"windowFunnelOrDefault" +"windowFunnelOrNull" +"windowFunnelResample" +"windowFunnelSimpleState" +"windowFunnelState" +"windowID" "WITH" +"WITH ADMIN OPTION" +"WITH CHECK" "WITH FILL" +"WITH GRANT OPTION" +"with_itemindex" +"WITH NAME" +"WITH REPLACE OPTION" "WITH TIES" "WK" "wkt" @@ -1463,14 +4825,23 @@ "wordShingleSimHashCaseInsensitive" "wordShingleSimHashCaseInsensitiveUTF8" "wordShingleSimHashUTF8" +"WRITABLE" "WW" +"wyHash64" "xor" +"xxh3" "xxHash32" "xxHash64" -"kostikConsistentHash" +"yandexConsistentHash" "YEAR" +"YEARS" "yearweek" "yesterday" "YY" "YYYY" +"YYYYMMDDhhmmssToDateTime" +"YYYYMMDDhhmmssToDateTime64" +"YYYYMMDDToDate" +"YYYYMMDDToDate32" +"ZKPATH" "zookeeperSessionUptime" diff --git a/tests/fuzz/dictionaries/datatypes.dict b/tests/fuzz/dictionaries/datatypes.dict index e4983ae5f68..232e89db0c0 100644 --- a/tests/fuzz/dictionaries/datatypes.dict +++ b/tests/fuzz/dictionaries/datatypes.dict @@ -1,185 +1,134 @@ -"Polygon" -"Ring" -"Point" -"SimpleAggregateFunction" -"MultiPolygon" -"IPv6" -"IntervalSecond" +"AggregateFunction" +"Array" +"BIGINT" +"BIGINT SIGNED" +"BIGINT UNSIGNED" +"BINARY" +"BINARY LARGE OBJECT" +"BINARY VARYING" +"BIT" +"BLOB" +"BYTE" +"BYTEA" +"Bool" +"CHAR" +"CHAR LARGE OBJECT" +"CHAR VARYING" +"CHARACTER" +"CHARACTER LARGE OBJECT" +"CHARACTER VARYING" +"CLOB" +"DEC" +"DOUBLE" +"DOUBLE PRECISION" +"Date" +"Date32" +"DateTime" +"DateTime32" +"DateTime64" +"Decimal" +"Decimal128" +"Decimal256" +"Decimal32" +"Decimal64" +"ENUM" +"Enum" +"Enum16" +"Enum8" +"FIXED" +"FLOAT" +"FixedString" +"Float32" +"Float64" +"GEOMETRY" +"INET4" +"INET6" +"INT" +"INT SIGNED" +"INT UNSIGNED" +"INT1" +"INT1 SIGNED" +"INT1 UNSIGNED" +"INTEGER" +"INTEGER SIGNED" +"INTEGER UNSIGNED" "IPv4" -"UInt32" -"IntervalYear" -"IntervalQuarter" -"IntervalMonth" +"IPv6" +"Int128" +"Int16" +"Int256" +"Int32" "Int64" +"Int8" "IntervalDay" "IntervalHour" -"Int16" -"UInt256" -"LowCardinality" -"AggregateFunction" -"Nothing" -"Decimal256" -"Tuple" -"Array" -"Enum16" +"IntervalMicrosecond" +"IntervalMillisecond" "IntervalMinute" -"FixedString" -"String" -"DateTime" -"Map" -"UUID" -"Decimal64" -"Nullable" -"Enum" -"Int32" -"UInt8" -"Date" -"Decimal32" -"UInt128" -"Float64" -"Nested" -"Int128" -"Decimal128" -"Int8" -"Decimal" -"Int256" -"DateTime64" -"Enum8" -"DateTime32" -"Date32" +"IntervalMonth" +"IntervalNanosecond" +"IntervalQuarter" +"IntervalSecond" "IntervalWeek" -"UInt64" -"UInt16" -"Float32" -"INET6" -"INET4" -"ENUM" -"BINARY" -"NATIONAL CHAR VARYING" -"BINARY VARYING" -"NCHAR LARGE OBJECT" -"NATIONAL CHARACTER VARYING" -"NATIONAL CHARACTER LARGE OBJECT" -"NATIONAL CHARACTER" -"NATIONAL CHAR" -"CHARACTER VARYING" +"IntervalYear" +"JSON" "LONGBLOB" -"MEDIUMTEXT" -"TEXT" -"TINYBLOB" -"VARCHAR2" -"CHARACTER LARGE OBJECT" -"DOUBLE PRECISION" "LONGTEXT" -"NVARCHAR" -"INT1 UNSIGNED" -"VARCHAR" -"CHAR VARYING" +"LowCardinality" "MEDIUMBLOB" -"NCHAR" -"CHAR" -"SMALLINT UNSIGNED" -"TIMESTAMP" -"FIXED" -"TINYTEXT" -"NUMERIC" -"DEC" -"TINYINT UNSIGNED" -"INTEGER UNSIGNED" -"INT UNSIGNED" -"CLOB" -"MEDIUMINT UNSIGNED" -"BOOL" -"SMALLINT" -"INTEGER SIGNED" -"NCHAR VARYING" -"INT SIGNED" -"TINYINT SIGNED" -"BIGINT SIGNED" -"BINARY LARGE OBJECT" -"SMALLINT SIGNED" "MEDIUMINT" -"INTEGER" -"INT1 SIGNED" -"BIGINT UNSIGNED" -"BYTEA" -"INT" -"SINGLE" -"FLOAT" "MEDIUMINT SIGNED" -"BOOLEAN" -"DOUBLE" -"INT1" -"CHAR LARGE OBJECT" -"TINYINT" -"BIGINT" -"CHARACTER" -"BYTE" -"BLOB" +"MEDIUMINT UNSIGNED" +"MEDIUMTEXT" +"Map" +"MultiPolygon" +"NATIONAL CHAR" +"NATIONAL CHAR VARYING" +"NATIONAL CHARACTER" +"NATIONAL CHARACTER LARGE OBJECT" +"NATIONAL CHARACTER VARYING" +"NCHAR" +"NCHAR LARGE OBJECT" +"NCHAR VARYING" +"NUMERIC" +"NVARCHAR" +"Nested" +"Nothing" +"Nullable" +"Object" +"Point" +"Polygon" "REAL" -"IPv6" -"IPv4" -"Enum" -"FixedString" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"Float64" -"String" -"String" -"UInt8" -"String" -"String" -"String" -"String" +"Ring" +"SET" +"SIGNED" +"SINGLE" +"SMALLINT" +"SMALLINT SIGNED" +"SMALLINT UNSIGNED" +"SimpleAggregateFunction" "String" +"TEXT" +"TIME" +"TIMESTAMP" +"TINYBLOB" +"TINYINT" +"TINYINT SIGNED" +"TINYINT UNSIGNED" +"TINYTEXT" +"Tuple" +"UInt128" "UInt16" -"DateTime" -"Decimal" -"String" -"Decimal" -"Decimal" -"UInt8" +"UInt256" "UInt32" -"UInt32" -"String" -"UInt32" -"Int8" -"Int16" -"Int32" -"String" -"Int32" -"Int8" -"Int64" -"String" -"Int16" -"Int32" -"Int32" -"Int8" "UInt64" -"String" -"Int32" -"Float32" -"Float32" -"Int32" -"Int8" -"Float64" -"Int8" -"String" -"Int8" -"Int64" -"String" -"Int8" -"String" -"Float32" +"UInt8" +"UNSIGNED" +"UUID" +"VARBINARY" +"VARCHAR" +"VARCHAR2" +"Variant" +"YEAR" +"bool" +"boolean" diff --git a/tests/fuzz/dictionaries/functions.dict b/tests/fuzz/dictionaries/functions.dict index 3fe8dbfe501..ec7f8017fb2 100644 --- a/tests/fuzz/dictionaries/functions.dict +++ b/tests/fuzz/dictionaries/functions.dict @@ -1,1134 +1,4172 @@ -"logTrace" -"aes_decrypt_mysql" -"aes_encrypt_mysql" -"decrypt" -"encrypt" -"bayesAB" -"snowflakeToDateTime64" -"snowflakeToDateTime" -"validateNestedArraySizes" -"wordShingleMinHashArgCaseInsensitiveUTF8" -"wordShingleMinHashArgUTF8" -"wordShingleMinHashArgCaseInsensitive" -"wordShingleMinHashArg" -"ngramMinHashArgCaseInsensitiveUTF8" -"ngramMinHashArgUTF8" -"ngramMinHashArgCaseInsensitive" -"wordShingleMinHashCaseInsensitiveUTF8" -"wordShingleMinHashUTF8" -"ngramMinHashCaseInsensitiveUTF8" -"ngramMinHashUTF8" -"ngramMinHashCaseInsensitive" -"wordShingleSimHashUTF8" -"ngramSimHashUTF8" -"ngramSimHashCaseInsensitive" -"ngramSimHash" -"toUnixTimestamp64Nano" -"toUnixTimestamp64Micro" -"jumpConsistentHash" -"kostikConsistentHash" -"addressToSymbol" -"toJSONString" -"JSON_VALUE" -"JSONExtractKeysAndValuesRaw" +"BIT_AND" +"BIT_ANDArgMax" +"BIT_ANDArgMin" +"BIT_ANDArray" +"BIT_ANDDistinct" +"BIT_ANDForEach" +"BIT_ANDIf" +"BIT_ANDMap" +"BIT_ANDMerge" +"BIT_ANDNull" +"BIT_ANDOrDefault" +"BIT_ANDOrNull" +"BIT_ANDResample" +"BIT_ANDSimpleState" +"BIT_ANDState" +"BIT_OR" +"BIT_ORArgMax" +"BIT_ORArgMin" +"BIT_ORArray" +"BIT_ORDistinct" +"BIT_ORForEach" +"BIT_ORIf" +"BIT_ORMap" +"BIT_ORMerge" +"BIT_ORNull" +"BIT_OROrDefault" +"BIT_OROrNull" +"BIT_ORResample" +"BIT_ORSimpleState" +"BIT_ORState" +"BIT_XOR" +"BIT_XORArgMax" +"BIT_XORArgMin" +"BIT_XORArray" +"BIT_XORDistinct" +"BIT_XORForEach" +"BIT_XORIf" +"BIT_XORMap" +"BIT_XORMerge" +"BIT_XORNull" +"BIT_XOROrDefault" +"BIT_XOROrNull" +"BIT_XORResample" +"BIT_XORSimpleState" +"BIT_XORState" +"BLAKE3" +"CAST" +"CHARACTER_LENGTH" +"CHAR_LENGTH" +"COVAR_POP" +"COVAR_POPArgMax" +"COVAR_POPArgMin" +"COVAR_POPArray" +"COVAR_POPDistinct" +"COVAR_POPForEach" +"COVAR_POPIf" +"COVAR_POPMap" +"COVAR_POPMerge" +"COVAR_POPNull" +"COVAR_POPOrDefault" +"COVAR_POPOrNull" +"COVAR_POPResample" +"COVAR_POPSimpleState" +"COVAR_POPState" +"COVAR_SAMP" +"COVAR_SAMPArgMax" +"COVAR_SAMPArgMin" +"COVAR_SAMPArray" +"COVAR_SAMPDistinct" +"COVAR_SAMPForEach" +"COVAR_SAMPIf" +"COVAR_SAMPMap" +"COVAR_SAMPMerge" +"COVAR_SAMPNull" +"COVAR_SAMPOrDefault" +"COVAR_SAMPOrNull" +"COVAR_SAMPResample" +"COVAR_SAMPSimpleState" +"COVAR_SAMPState" +"CRC32" +"CRC32IEEE" +"CRC64" +"DATABASE" +"DATE" +"DATE_DIFF" +"DATE_FORMAT" +"DATE_TRUNC" +"DAY" +"DAYOFMONTH" +"DAYOFWEEK" +"DAYOFYEAR" +"FORMAT_BYTES" +"FQDN" +"FROM_BASE64" +"FROM_DAYS" +"FROM_UNIXTIME" +"HOUR" +"INET6_ATON" +"INET6_NTOA" +"INET_ATON" +"INET_NTOA" +"IPv4CIDRToRange" +"IPv4NumToString" +"IPv4NumToStringClassC" +"IPv4StringToNum" +"IPv4StringToNumOrDefault" +"IPv4StringToNumOrNull" +"IPv4ToIPv6" +"IPv6CIDRToRange" +"IPv6NumToString" +"IPv6StringToNum" +"IPv6StringToNumOrDefault" +"IPv6StringToNumOrNull" +"JSONArrayLength" +"JSONExtract" +"JSONExtractArrayRaw" +"JSONExtractBool" +"JSONExtractFloat" +"JSONExtractInt" "JSONExtractKeys" +"JSONExtractKeysAndValues" +"JSONExtractKeysAndValuesRaw" +"JSONExtractRaw" "JSONExtractString" -"JSONType" +"JSONExtractUInt" +"JSONHas" "JSONKey" "JSONLength" -"isValidJSON" -"isZeroOrNull" -"assumeNotNull" -"s2CapUnion" -"s2CapContains" -"s2CellsIntersect" -"s2GetNeighbors" -"s2ToGeo" -"h3GetFaces" -"h3IsResClassIII" -"h3ToString" -"stringToH3" -"h3ToParent" -"h3GetResolution" -"h3EdgeLengthM" -"svg" -"SVG" -"equals" -"geohashesInBox" -"polygonsIntersectionCartesian" -"polygonPerimeterSpherical" -"bitHammingDistance" -"polygonsDistanceSpherical" -"polygonsSymDifferenceSpherical" -"polygonAreaSpherical" -"greatCircleDistance" -"toInt256" -"mapKeys" -"sign" -"reinterpretAsUInt8" -"atanh" -"formatReadableTimeDelta" -"parseTimeDelta" -"geohashEncode" -"atan2" -"acos" +"JSONType" +"JSON_ARRAY_LENGTH" +"JSON_EXISTS" +"JSON_QUERY" +"JSON_VALUE" +"L1Distance" +"L1Norm" +"L1Normalize" +"L2Distance" +"L2Norm" +"L2Normalize" +"L2SquaredDistance" +"L2SquaredNorm" +"LAST_DAY" +"LinfDistance" +"LinfNorm" +"LinfNormalize" +"LpDistance" +"LpNorm" +"LpNormalize" +"MACNumToString" +"MACStringToNum" +"MACStringToOUI" +"MAP_FROM_ARRAYS" +"MD4" +"MD5" +"MILLISECOND" +"MINUTE" +"MONTH" +"OCTET_LENGTH" +"QUARTER" +"REGEXP_EXTRACT" +"REGEXP_MATCHES" +"REGEXP_REPLACE" +"SCHEMA" +"SECOND" +"SHA1" "SHA224" -"visitParamExtractBool" -"h3kRing" -"asin" -"sin" -"reinterpretAsUInt128" -"lgamma" -"log2" -"geoToH3" -"toUnixTimestamp64Milli" -"firstSignificantSubdomainCustom" -"simpleJSONExtractBool" -"visitParamExtractFloat" -"visitParamExtractInt" -"simpleJSONHas" -"visitParamHas" -"sigmoid" -"cutQueryStringAndFragment" -"dictGetInt16OrDefault" -"cutToFirstSignificantSubdomainCustom" +"SHA256" +"SHA384" +"SHA512" +"SHA512_256" +"STD" +"STDArgMax" +"STDArgMin" +"STDArray" +"STDDEV_POP" +"STDDEV_POPArgMax" +"STDDEV_POPArgMin" +"STDDEV_POPArray" +"STDDEV_POPDistinct" +"STDDEV_POPForEach" +"STDDEV_POPIf" +"STDDEV_POPMap" +"STDDEV_POPMerge" +"STDDEV_POPNull" +"STDDEV_POPOrDefault" +"STDDEV_POPOrNull" +"STDDEV_POPResample" +"STDDEV_POPSimpleState" +"STDDEV_POPState" +"STDDEV_SAMP" +"STDDEV_SAMPArgMax" +"STDDEV_SAMPArgMin" +"STDDEV_SAMPArray" +"STDDEV_SAMPDistinct" +"STDDEV_SAMPForEach" +"STDDEV_SAMPIf" +"STDDEV_SAMPMap" +"STDDEV_SAMPMerge" +"STDDEV_SAMPNull" +"STDDEV_SAMPOrDefault" +"STDDEV_SAMPOrNull" +"STDDEV_SAMPResample" +"STDDEV_SAMPSimpleState" +"STDDEV_SAMPState" +"STDDistinct" +"STDForEach" +"STDIf" +"STDMap" +"STDMerge" +"STDNull" +"STDOrDefault" +"STDOrNull" +"STDResample" +"STDSimpleState" +"STDState" +"SUBSTRING_INDEX" +"SVG" +"TIMESTAMP_DIFF" +"TO_BASE64" +"TO_DAYS" +"TO_UNIXTIME" +"ULIDStringToDateTime" +"URLHash" "URLHierarchy" -"extractURLParameterNames" -"toDecimal128OrZero" -"extractURLParameters" -"path" -"pathFull" -"port" -"domain" +"URLPathHierarchy" +"UTCTimestamp" +"UTC_timestamp" +"UUIDNumToString" +"UUIDStringToNum" +"VAR_POP" +"VAR_POPArgMax" +"VAR_POPArgMin" +"VAR_POPArray" +"VAR_POPDistinct" +"VAR_POPForEach" +"VAR_POPIf" +"VAR_POPMap" +"VAR_POPMerge" +"VAR_POPNull" +"VAR_POPOrDefault" +"VAR_POPOrNull" +"VAR_POPResample" +"VAR_POPSimpleState" +"VAR_POPState" +"VAR_SAMP" +"VAR_SAMPArgMax" +"VAR_SAMPArgMin" +"VAR_SAMPArray" +"VAR_SAMPDistinct" +"VAR_SAMPForEach" +"VAR_SAMPIf" +"VAR_SAMPMap" +"VAR_SAMPMerge" +"VAR_SAMPNull" +"VAR_SAMPOrDefault" +"VAR_SAMPOrNull" +"VAR_SAMPResample" +"VAR_SAMPSimpleState" +"VAR_SAMPState" +"YEAR" +"YYYYMMDDToDate" +"YYYYMMDDToDate32" +"YYYYMMDDhhmmssToDateTime" +"YYYYMMDDhhmmssToDateTime64" +"_CAST" +"__bitBoolMaskAnd" +"__bitBoolMaskOr" +"__bitSwapLastTwo" +"__bitWrapperFunc" +"__getScalar" +"__scalarSubqueryResult" +"abs" +"accurateCast" +"accurateCastOrDefault" +"accurateCastOrNull" +"acos" +"acosh" +"addDate" +"addDays" +"addHours" +"addInterval" +"addMicroseconds" +"addMilliseconds" +"addMinutes" +"addMonths" +"addNanoseconds" +"addQuarters" +"addSeconds" +"addTupleOfIntervals" +"addWeeks" +"addYears" +"addressToLine" +"addressToLineWithInlines" +"addressToSymbol" +"aes_decrypt_mysql" +"aes_encrypt_mysql" +"age" +"aggThrow" +"aggThrowArgMax" +"aggThrowArgMin" +"aggThrowArray" +"aggThrowDistinct" +"aggThrowForEach" +"aggThrowIf" +"aggThrowMap" +"aggThrowMerge" +"aggThrowNull" +"aggThrowOrDefault" +"aggThrowOrNull" +"aggThrowResample" +"aggThrowSimpleState" +"aggThrowState" +"alphaTokens" +"analysisOfVariance" +"analysisOfVarianceArgMax" +"analysisOfVarianceArgMin" +"analysisOfVarianceArray" +"analysisOfVarianceDistinct" +"analysisOfVarianceForEach" +"analysisOfVarianceIf" +"analysisOfVarianceMap" +"analysisOfVarianceMerge" +"analysisOfVarianceNull" +"analysisOfVarianceOrDefault" +"analysisOfVarianceOrNull" +"analysisOfVarianceResample" +"analysisOfVarianceSimpleState" +"analysisOfVarianceState" +"and" +"anova" +"anovaArgMax" +"anovaArgMin" +"anovaArray" +"anovaDistinct" +"anovaForEach" +"anovaIf" +"anovaMap" +"anovaMerge" +"anovaNull" +"anovaOrDefault" +"anovaOrNull" +"anovaResample" +"anovaSimpleState" +"anovaState" +"any" +"anyArgMax" +"anyArgMin" +"anyArray" +"anyDistinct" +"anyForEach" +"anyHeavy" +"anyHeavyArgMax" +"anyHeavyArgMin" +"anyHeavyArray" +"anyHeavyDistinct" +"anyHeavyForEach" +"anyHeavyIf" +"anyHeavyMap" +"anyHeavyMerge" +"anyHeavyNull" +"anyHeavyOrDefault" +"anyHeavyOrNull" +"anyHeavyResample" +"anyHeavySimpleState" +"anyHeavyState" +"anyIf" +"anyLast" +"anyLastArgMax" +"anyLastArgMin" +"anyLastArray" +"anyLastDistinct" +"anyLastForEach" +"anyLastIf" +"anyLastMap" +"anyLastMerge" +"anyLastNull" +"anyLastOrDefault" +"anyLastOrNull" +"anyLastResample" +"anyLastSimpleState" +"anyLastState" +"anyLast_respect_nulls" +"anyLast_respect_nullsArgMax" +"anyLast_respect_nullsArgMin" +"anyLast_respect_nullsArray" +"anyLast_respect_nullsDistinct" +"anyLast_respect_nullsForEach" +"anyLast_respect_nullsIf" +"anyLast_respect_nullsMap" +"anyLast_respect_nullsMerge" +"anyLast_respect_nullsNull" +"anyLast_respect_nullsOrDefault" +"anyLast_respect_nullsOrNull" +"anyLast_respect_nullsResample" +"anyLast_respect_nullsSimpleState" +"anyLast_respect_nullsState" +"anyMap" +"anyMerge" +"anyNull" +"anyOrDefault" +"anyOrNull" +"anyResample" +"anySimpleState" +"anyState" +"any_respect_nulls" +"any_respect_nullsArgMax" +"any_respect_nullsArgMin" +"any_respect_nullsArray" +"any_respect_nullsDistinct" +"any_respect_nullsForEach" +"any_respect_nullsIf" +"any_respect_nullsMap" +"any_respect_nullsMerge" +"any_respect_nullsNull" +"any_respect_nullsOrDefault" +"any_respect_nullsOrNull" +"any_respect_nullsResample" +"any_respect_nullsSimpleState" +"any_respect_nullsState" +"any_value" +"any_valueArgMax" +"any_valueArgMin" +"any_valueArray" +"any_valueDistinct" +"any_valueForEach" +"any_valueIf" +"any_valueMap" +"any_valueMerge" +"any_valueNull" +"any_valueOrDefault" +"any_valueOrNull" +"any_valueResample" +"any_valueSimpleState" +"any_valueState" +"any_value_respect_nulls" +"any_value_respect_nullsArgMax" +"any_value_respect_nullsArgMin" +"any_value_respect_nullsArray" +"any_value_respect_nullsDistinct" +"any_value_respect_nullsForEach" +"any_value_respect_nullsIf" +"any_value_respect_nullsMap" +"any_value_respect_nullsMerge" +"any_value_respect_nullsNull" +"any_value_respect_nullsOrDefault" +"any_value_respect_nullsOrNull" +"any_value_respect_nullsResample" +"any_value_respect_nullsSimpleState" +"any_value_respect_nullsState" +"appendTrailingCharIfAbsent" +"approx_top_count" +"approx_top_countArgMax" +"approx_top_countArgMin" +"approx_top_countArray" +"approx_top_countDistinct" +"approx_top_countForEach" +"approx_top_countIf" +"approx_top_countMap" +"approx_top_countMerge" +"approx_top_countNull" +"approx_top_countOrDefault" +"approx_top_countOrNull" +"approx_top_countResample" +"approx_top_countSimpleState" +"approx_top_countState" +"approx_top_k" +"approx_top_kArgMax" +"approx_top_kArgMin" +"approx_top_kArray" +"approx_top_kDistinct" +"approx_top_kForEach" +"approx_top_kIf" +"approx_top_kMap" +"approx_top_kMerge" +"approx_top_kNull" +"approx_top_kOrDefault" +"approx_top_kOrNull" +"approx_top_kResample" +"approx_top_kSimpleState" +"approx_top_kState" +"approx_top_sum" +"approx_top_sumArgMax" +"approx_top_sumArgMin" +"approx_top_sumArray" +"approx_top_sumDistinct" +"approx_top_sumForEach" +"approx_top_sumIf" +"approx_top_sumMap" +"approx_top_sumMerge" +"approx_top_sumNull" +"approx_top_sumOrDefault" +"approx_top_sumOrNull" +"approx_top_sumResample" +"approx_top_sumSimpleState" +"approx_top_sumState" +"argMax" +"argMaxArgMax" +"argMaxArgMin" +"argMaxArray" +"argMaxDistinct" +"argMaxForEach" +"argMaxIf" +"argMaxMap" +"argMaxMerge" +"argMaxNull" +"argMaxOrDefault" +"argMaxOrNull" +"argMaxResample" +"argMaxSimpleState" +"argMaxState" +"argMin" +"argMinArgMax" +"argMinArgMin" +"argMinArray" +"argMinDistinct" +"argMinForEach" +"argMinIf" +"argMinMap" +"argMinMerge" +"argMinNull" +"argMinOrDefault" +"argMinOrNull" +"argMinResample" +"argMinSimpleState" +"argMinState" +"array" +"arrayAUC" +"arrayAll" +"arrayAvg" +"arrayCompact" +"arrayConcat" +"arrayCount" +"arrayCumSum" +"arrayCumSumNonNegative" +"arrayDifference" +"arrayDistinct" +"arrayDotProduct" +"arrayElement" +"arrayEnumerate" +"arrayEnumerateDense" +"arrayEnumerateDenseRanked" +"arrayEnumerateUniq" +"arrayEnumerateUniqRanked" +"arrayExists" +"arrayFill" +"arrayFilter" +"arrayFirst" +"arrayFirstIndex" +"arrayFirstOrNull" +"arrayFlatten" +"arrayFold" +"arrayIntersect" +"arrayJaccardIndex" +"arrayJoin" +"arrayLast" +"arrayLastIndex" +"arrayLastOrNull" +"arrayMap" +"arrayMax" +"arrayMin" +"arrayPartialReverseSort" +"arrayPartialShuffle" +"arrayPartialSort" +"arrayPopBack" +"arrayPopFront" +"arrayProduct" +"arrayPushBack" +"arrayPushFront" +"arrayRandomSample" +"arrayReduce" +"arrayReduceInRanges" +"arrayResize" +"arrayReverse" +"arrayReverseFill" +"arrayReverseSort" +"arrayReverseSplit" +"arrayRotateLeft" +"arrayRotateRight" +"arrayShiftLeft" +"arrayShiftRight" +"arrayShingles" +"arrayShuffle" +"arraySlice" +"arraySort" +"arraySplit" +"arrayStringConcat" +"arraySum" +"arrayUniq" +"arrayWithConstant" +"arrayZip" +"array_agg" +"array_aggArgMax" +"array_aggArgMin" +"array_aggArray" +"array_aggDistinct" +"array_aggForEach" +"array_aggIf" +"array_aggMap" +"array_aggMerge" +"array_aggNull" +"array_aggOrDefault" +"array_aggOrNull" +"array_aggResample" +"array_aggSimpleState" +"array_aggState" +"array_concat_agg" +"array_concat_aggArgMax" +"array_concat_aggArgMin" +"array_concat_aggArray" +"array_concat_aggDistinct" +"array_concat_aggForEach" +"array_concat_aggIf" +"array_concat_aggMap" +"array_concat_aggMerge" +"array_concat_aggNull" +"array_concat_aggOrDefault" +"array_concat_aggOrNull" +"array_concat_aggResample" +"array_concat_aggSimpleState" +"array_concat_aggState" +"ascii" +"asin" +"asinh" +"assumeNotNull" +"atan" +"atan2" +"atanh" +"avg" +"avgArgMax" +"avgArgMin" +"avgArray" +"avgDistinct" +"avgForEach" +"avgIf" +"avgMap" +"avgMerge" +"avgNull" +"avgOrDefault" +"avgOrNull" +"avgResample" +"avgSimpleState" +"avgState" +"avgWeighted" +"avgWeightedArgMax" +"avgWeightedArgMin" +"avgWeightedArray" +"avgWeightedDistinct" +"avgWeightedForEach" +"avgWeightedIf" +"avgWeightedMap" +"avgWeightedMerge" +"avgWeightedNull" +"avgWeightedOrDefault" +"avgWeightedOrNull" +"avgWeightedResample" +"avgWeightedSimpleState" +"avgWeightedState" +"bar" +"base58Decode" +"base58Encode" +"base64Decode" +"base64Encode" +"basename" +"bin" +"bitAnd" +"bitCount" +"bitHammingDistance" +"bitNot" +"bitOr" +"bitPositionsToArray" +"bitRotateLeft" +"bitRotateRight" +"bitShiftLeft" +"bitShiftRight" +"bitSlice" "bitTest" -"ngramSearchCaseInsensitiveUTF8" +"bitTestAll" +"bitTestAny" +"bitXor" +"bitmapAnd" +"bitmapAndCardinality" +"bitmapAndnot" +"bitmapAndnotCardinality" +"bitmapBuild" +"bitmapCardinality" +"bitmapContains" +"bitmapHasAll" +"bitmapHasAny" "bitmapMax" -"ngramDistanceCaseInsensitiveUTF8" -"ngramDistance" +"bitmapMin" +"bitmapOr" +"bitmapOrCardinality" +"bitmapSubsetInRange" +"bitmapSubsetLimit" +"bitmapToArray" +"bitmapTransform" +"bitmapXor" +"bitmapXorCardinality" +"bitmaskToArray" +"bitmaskToList" +"blockNumber" +"blockSerializedSize" +"blockSize" +"boundingRatio" +"boundingRatioArgMax" +"boundingRatioArgMin" +"boundingRatioArray" +"boundingRatioDistinct" +"boundingRatioForEach" +"boundingRatioIf" +"boundingRatioMap" +"boundingRatioMerge" +"boundingRatioNull" +"boundingRatioOrDefault" +"boundingRatioOrNull" +"boundingRatioResample" +"boundingRatioSimpleState" +"boundingRatioState" +"buildId" +"byteHammingDistance" +"byteSize" +"byteSlice" +"byteSwap" +"caseWithExpr" +"caseWithExpression" +"caseWithoutExpr" +"caseWithoutExpression" +"catboostEvaluate" +"categoricalInformationValue" +"categoricalInformationValueArgMax" +"categoricalInformationValueArgMin" +"categoricalInformationValueArray" +"categoricalInformationValueDistinct" +"categoricalInformationValueForEach" +"categoricalInformationValueIf" +"categoricalInformationValueMap" +"categoricalInformationValueMerge" +"categoricalInformationValueNull" +"categoricalInformationValueOrDefault" +"categoricalInformationValueOrNull" +"categoricalInformationValueResample" +"categoricalInformationValueSimpleState" +"categoricalInformationValueState" +"cbrt" +"ceil" +"ceiling" +"char" +"cityHash64" +"coalesce" +"concat" +"concatAssumeInjective" +"concatWithSeparator" +"concatWithSeparatorAssumeInjective" +"concat_ws" +"connectionId" +"connection_id" +"contingency" +"contingencyArgMax" +"contingencyArgMin" +"contingencyArray" +"contingencyDistinct" +"contingencyForEach" +"contingencyIf" +"contingencyMap" +"contingencyMerge" +"contingencyNull" +"contingencyOrDefault" +"contingencyOrNull" +"contingencyResample" +"contingencySimpleState" +"contingencyState" +"convertCharset" +"corr" +"corrArgMax" +"corrArgMin" +"corrArray" +"corrDistinct" +"corrForEach" +"corrIf" +"corrMap" +"corrMatrix" +"corrMatrixArgMax" +"corrMatrixArgMin" +"corrMatrixArray" +"corrMatrixDistinct" +"corrMatrixForEach" +"corrMatrixIf" +"corrMatrixMap" +"corrMatrixMerge" +"corrMatrixNull" +"corrMatrixOrDefault" +"corrMatrixOrNull" +"corrMatrixResample" +"corrMatrixSimpleState" +"corrMatrixState" +"corrMerge" +"corrNull" +"corrOrDefault" +"corrOrNull" +"corrResample" +"corrSimpleState" +"corrStable" +"corrStableArgMax" +"corrStableArgMin" +"corrStableArray" +"corrStableDistinct" +"corrStableForEach" +"corrStableIf" +"corrStableMap" +"corrStableMerge" +"corrStableNull" +"corrStableOrDefault" +"corrStableOrNull" +"corrStableResample" +"corrStableSimpleState" +"corrStableState" +"corrState" +"cos" +"cosh" +"cosineDistance" +"count" +"countArgMax" +"countArgMin" +"countArray" +"countDigits" +"countDistinct" +"countEqual" +"countForEach" +"countIf" +"countMap" +"countMatches" +"countMatchesCaseInsensitive" +"countMerge" +"countNull" +"countOrDefault" +"countOrNull" +"countResample" +"countSimpleState" +"countState" +"countSubstrings" +"countSubstringsCaseInsensitive" +"countSubstringsCaseInsensitiveUTF8" +"covarPop" +"covarPopArgMax" +"covarPopArgMin" +"covarPopArray" +"covarPopDistinct" +"covarPopForEach" +"covarPopIf" +"covarPopMap" +"covarPopMatrix" +"covarPopMatrixArgMax" +"covarPopMatrixArgMin" +"covarPopMatrixArray" +"covarPopMatrixDistinct" +"covarPopMatrixForEach" +"covarPopMatrixIf" +"covarPopMatrixMap" +"covarPopMatrixMerge" +"covarPopMatrixNull" +"covarPopMatrixOrDefault" +"covarPopMatrixOrNull" +"covarPopMatrixResample" +"covarPopMatrixSimpleState" +"covarPopMatrixState" +"covarPopMerge" +"covarPopNull" +"covarPopOrDefault" +"covarPopOrNull" +"covarPopResample" +"covarPopSimpleState" +"covarPopStable" +"covarPopStableArgMax" +"covarPopStableArgMin" +"covarPopStableArray" +"covarPopStableDistinct" +"covarPopStableForEach" +"covarPopStableIf" +"covarPopStableMap" +"covarPopStableMerge" +"covarPopStableNull" +"covarPopStableOrDefault" +"covarPopStableOrNull" +"covarPopStableResample" +"covarPopStableSimpleState" +"covarPopStableState" +"covarPopState" +"covarSamp" +"covarSampArgMax" +"covarSampArgMin" +"covarSampArray" +"covarSampDistinct" +"covarSampForEach" +"covarSampIf" +"covarSampMap" +"covarSampMatrix" +"covarSampMatrixArgMax" +"covarSampMatrixArgMin" +"covarSampMatrixArray" +"covarSampMatrixDistinct" +"covarSampMatrixForEach" +"covarSampMatrixIf" +"covarSampMatrixMap" +"covarSampMatrixMerge" +"covarSampMatrixNull" +"covarSampMatrixOrDefault" +"covarSampMatrixOrNull" +"covarSampMatrixResample" +"covarSampMatrixSimpleState" +"covarSampMatrixState" +"covarSampMerge" +"covarSampNull" +"covarSampOrDefault" +"covarSampOrNull" +"covarSampResample" +"covarSampSimpleState" +"covarSampStable" +"covarSampStableArgMax" +"covarSampStableArgMin" +"covarSampStableArray" +"covarSampStableDistinct" +"covarSampStableForEach" +"covarSampStableIf" +"covarSampStableMap" +"covarSampStableMerge" +"covarSampStableNull" +"covarSampStableOrDefault" +"covarSampStableOrNull" +"covarSampStableResample" +"covarSampStableSimpleState" +"covarSampStableState" +"covarSampState" +"cramersV" +"cramersVArgMax" +"cramersVArgMin" +"cramersVArray" +"cramersVBiasCorrected" +"cramersVBiasCorrectedArgMax" +"cramersVBiasCorrectedArgMin" +"cramersVBiasCorrectedArray" +"cramersVBiasCorrectedDistinct" +"cramersVBiasCorrectedForEach" +"cramersVBiasCorrectedIf" +"cramersVBiasCorrectedMap" +"cramersVBiasCorrectedMerge" +"cramersVBiasCorrectedNull" +"cramersVBiasCorrectedOrDefault" +"cramersVBiasCorrectedOrNull" +"cramersVBiasCorrectedResample" +"cramersVBiasCorrectedSimpleState" +"cramersVBiasCorrectedState" +"cramersVDistinct" +"cramersVForEach" +"cramersVIf" +"cramersVMap" +"cramersVMerge" +"cramersVNull" +"cramersVOrDefault" +"cramersVOrNull" +"cramersVResample" +"cramersVSimpleState" +"cramersVState" +"curdate" +"currentDatabase" +"currentProfiles" +"currentRoles" +"currentSchemas" +"currentUser" +"current_database" +"current_date" +"current_schemas" +"current_timestamp" +"cutFragment" +"cutIPv6" +"cutQueryString" +"cutQueryStringAndFragment" +"cutToFirstSignificantSubdomain" +"cutToFirstSignificantSubdomainCustom" +"cutToFirstSignificantSubdomainCustomRFC" +"cutToFirstSignificantSubdomainCustomWithWWW" +"cutToFirstSignificantSubdomainCustomWithWWWRFC" +"cutToFirstSignificantSubdomainRFC" +"cutToFirstSignificantSubdomainWithWWW" +"cutToFirstSignificantSubdomainWithWWWRFC" +"cutURLParameter" +"cutWWW" +"damerauLevenshteinDistance" +"dateDiff" +"dateName" +"dateTime64ToSnowflake" +"dateTimeToSnowflake" +"dateTrunc" +"date_diff" +"decodeHTMLComponent" +"decodeURLComponent" +"decodeURLFormComponent" +"decodeXMLComponent" +"decrypt" +"defaultProfiles" +"defaultRoles" +"defaultValueOfArgumentType" +"defaultValueOfTypeName" +"degrees" +"deltaSum" +"deltaSumArgMax" +"deltaSumArgMin" +"deltaSumArray" +"deltaSumDistinct" +"deltaSumForEach" +"deltaSumIf" +"deltaSumMap" +"deltaSumMerge" +"deltaSumNull" +"deltaSumOrDefault" +"deltaSumOrNull" +"deltaSumResample" +"deltaSumSimpleState" +"deltaSumState" +"deltaSumTimestamp" +"deltaSumTimestampArgMax" +"deltaSumTimestampArgMin" +"deltaSumTimestampArray" +"deltaSumTimestampDistinct" +"deltaSumTimestampForEach" +"deltaSumTimestampIf" +"deltaSumTimestampMap" +"deltaSumTimestampMerge" +"deltaSumTimestampNull" +"deltaSumTimestampOrDefault" +"deltaSumTimestampOrNull" +"deltaSumTimestampResample" +"deltaSumTimestampSimpleState" +"deltaSumTimestampState" +"demangle" +"dense_rank" +"dense_rankArgMax" +"dense_rankArgMin" +"dense_rankArray" +"dense_rankDistinct" +"dense_rankForEach" +"dense_rankIf" +"dense_rankMap" +"dense_rankMerge" +"dense_rankNull" +"dense_rankOrDefault" +"dense_rankOrNull" +"dense_rankResample" +"dense_rankSimpleState" +"dense_rankState" +"detectCharset" +"detectLanguage" +"detectLanguageMixed" +"detectLanguageUnknown" +"detectProgrammingLanguage" +"detectTonality" +"dictGet" +"dictGetAll" +"dictGetChildren" +"dictGetDate" +"dictGetDateOrDefault" +"dictGetDateTime" +"dictGetDateTimeOrDefault" +"dictGetDescendants" +"dictGetFloat32" +"dictGetFloat32OrDefault" +"dictGetFloat64" +"dictGetFloat64OrDefault" +"dictGetHierarchy" +"dictGetIPv4" +"dictGetIPv4OrDefault" +"dictGetIPv6" +"dictGetIPv6OrDefault" +"dictGetInt16" +"dictGetInt16OrDefault" +"dictGetInt32" +"dictGetInt32OrDefault" +"dictGetInt64" +"dictGetInt64OrDefault" +"dictGetInt8" +"dictGetInt8OrDefault" +"dictGetOrDefault" +"dictGetOrNull" +"dictGetString" +"dictGetStringOrDefault" +"dictGetUInt16" +"dictGetUInt16OrDefault" +"dictGetUInt32" +"dictGetUInt32OrDefault" +"dictGetUInt64" +"dictGetUInt64OrDefault" +"dictGetUInt8" +"dictGetUInt8OrDefault" +"dictGetUUID" +"dictGetUUIDOrDefault" +"dictHas" +"dictIsIn" +"displayName" +"distanceL1" +"distanceL2" +"distanceL2Squared" +"distanceLinf" +"distanceLp" +"divide" +"divideDecimal" +"domain" +"domainRFC" +"domainWithoutWWW" +"domainWithoutWWWRFC" +"dotProduct" +"dumpColumnStructure" +"e" +"editDistance" +"empty" +"emptyArrayDate" +"emptyArrayDateTime" +"emptyArrayFloat32" +"emptyArrayFloat64" +"emptyArrayInt16" +"emptyArrayInt32" +"emptyArrayInt64" +"emptyArrayInt8" +"emptyArrayString" +"emptyArrayToSingle" +"emptyArrayUInt16" +"emptyArrayUInt32" +"emptyArrayUInt64" +"emptyArrayUInt8" +"enabledProfiles" +"enabledRoles" +"encodeURLComponent" +"encodeURLFormComponent" +"encodeXMLComponent" +"encrypt" +"endsWith" +"endsWithUTF8" +"entropy" +"entropyArgMax" +"entropyArgMin" +"entropyArray" +"entropyDistinct" +"entropyForEach" +"entropyIf" +"entropyMap" +"entropyMerge" +"entropyNull" +"entropyOrDefault" +"entropyOrNull" +"entropyResample" +"entropySimpleState" +"entropyState" +"equals" +"erf" +"erfc" +"errorCodeToName" +"evalMLMethod" +"exp" +"exp10" +"exp2" +"exponentialMovingAverage" +"exponentialMovingAverageArgMax" +"exponentialMovingAverageArgMin" +"exponentialMovingAverageArray" +"exponentialMovingAverageDistinct" +"exponentialMovingAverageForEach" +"exponentialMovingAverageIf" +"exponentialMovingAverageMap" +"exponentialMovingAverageMerge" +"exponentialMovingAverageNull" +"exponentialMovingAverageOrDefault" +"exponentialMovingAverageOrNull" +"exponentialMovingAverageResample" +"exponentialMovingAverageSimpleState" +"exponentialMovingAverageState" +"exponentialTimeDecayedAvg" +"exponentialTimeDecayedAvgArgMax" +"exponentialTimeDecayedAvgArgMin" +"exponentialTimeDecayedAvgArray" +"exponentialTimeDecayedAvgDistinct" +"exponentialTimeDecayedAvgForEach" +"exponentialTimeDecayedAvgIf" +"exponentialTimeDecayedAvgMap" +"exponentialTimeDecayedAvgMerge" +"exponentialTimeDecayedAvgNull" +"exponentialTimeDecayedAvgOrDefault" +"exponentialTimeDecayedAvgOrNull" +"exponentialTimeDecayedAvgResample" +"exponentialTimeDecayedAvgSimpleState" +"exponentialTimeDecayedAvgState" +"exponentialTimeDecayedCount" +"exponentialTimeDecayedCountArgMax" +"exponentialTimeDecayedCountArgMin" +"exponentialTimeDecayedCountArray" +"exponentialTimeDecayedCountDistinct" +"exponentialTimeDecayedCountForEach" +"exponentialTimeDecayedCountIf" +"exponentialTimeDecayedCountMap" +"exponentialTimeDecayedCountMerge" +"exponentialTimeDecayedCountNull" +"exponentialTimeDecayedCountOrDefault" +"exponentialTimeDecayedCountOrNull" +"exponentialTimeDecayedCountResample" +"exponentialTimeDecayedCountSimpleState" +"exponentialTimeDecayedCountState" +"exponentialTimeDecayedMax" +"exponentialTimeDecayedMaxArgMax" +"exponentialTimeDecayedMaxArgMin" +"exponentialTimeDecayedMaxArray" +"exponentialTimeDecayedMaxDistinct" +"exponentialTimeDecayedMaxForEach" +"exponentialTimeDecayedMaxIf" +"exponentialTimeDecayedMaxMap" +"exponentialTimeDecayedMaxMerge" +"exponentialTimeDecayedMaxNull" +"exponentialTimeDecayedMaxOrDefault" +"exponentialTimeDecayedMaxOrNull" +"exponentialTimeDecayedMaxResample" +"exponentialTimeDecayedMaxSimpleState" +"exponentialTimeDecayedMaxState" +"exponentialTimeDecayedSum" +"exponentialTimeDecayedSumArgMax" +"exponentialTimeDecayedSumArgMin" +"exponentialTimeDecayedSumArray" +"exponentialTimeDecayedSumDistinct" +"exponentialTimeDecayedSumForEach" +"exponentialTimeDecayedSumIf" +"exponentialTimeDecayedSumMap" +"exponentialTimeDecayedSumMerge" +"exponentialTimeDecayedSumNull" +"exponentialTimeDecayedSumOrDefault" +"exponentialTimeDecayedSumOrNull" +"exponentialTimeDecayedSumResample" +"exponentialTimeDecayedSumSimpleState" +"exponentialTimeDecayedSumState" +"extract" +"extractAll" +"extractAllGroups" "extractAllGroupsHorizontal" "extractAllGroupsVertical" -"multiFuzzyMatchAllIndices" -"multiFuzzyMatchAnyIndex" -"multiMatchAnyIndex" -"replaceRegexpAll" -"emptyArrayInt16" -"dictGetOrDefault" -"replaceOne" -"emptyArrayInt32" -"extract" -"readWKTPolygon" -"notILike" -"geohashDecode" -"toModifiedJulianDay" -"notLike" -"countSubstringsCaseInsensitive" -"IPv4StringToNum" -"positionUTF8" -"hasToken" -"dictGetDate" -"multiSearchFirstPosition" -"multiSearchFirstIndexCaseInsensitiveUTF8" -"atan" -"rowNumberInAllBlocks" -"multiSearchFirstIndexCaseInsensitive" -"multiSearchAnyUTF8" -"parseDateTime32BestEffortOrZero" -"blockNumber" -"cutURLParameter" -"multiSearchFirstPositionCaseInsensitive" -"positionCaseInsensitiveUTF8" -"positionCaseInsensitive" -"splitByRegexp" -"toInt256OrZero" -"pointInPolygon" -"splitByWhitespace" -"lemmatize" -"synonyms" -"tryBase64Decode" -"multiSearchAllPositions" -"FROM_BASE64" -"replaceRegexpOne" -"subBitmap" -"defaultValueOfArgumentType" -"base64Decode" -"regionToContinent" -"ilike" -"fromModifiedJulianDay" -"TO_BASE64" -"dictIsIn" -"base64Encode" -"decodeXMLComponent" -"countMatchesCaseInsensitive" -"normalizeQuery" -"erf" -"trimBoth" -"fromUnixTimestamp64Nano" -"lessOrEquals" -"subtractQuarters" -"ngramSearch" -"readWKTRing" -"trimRight" -"endsWith" -"ngramDistanceCaseInsensitive" -"connectionId" -"initialQueryID" -"startsWith" -"tan" -"substring" +"extractGroups" +"extractKeyValuePairs" +"extractKeyValuePairsWithEscaping" +"extractTextFromHTML" +"extractURLParameter" +"extractURLParameterNames" +"extractURLParameters" +"factorial" +"farmFingerprint64" +"farmHash64" +"file" +"filesystemAvailable" +"filesystemCapacity" +"filesystemUnreserved" +"finalizeAggregation" +"firstLine" +"firstSignificantSubdomain" +"firstSignificantSubdomainCustom" +"firstSignificantSubdomainCustomRFC" +"firstSignificantSubdomainRFC" +"first_value" +"first_valueArgMax" +"first_valueArgMin" +"first_valueArray" +"first_valueDistinct" +"first_valueForEach" +"first_valueIf" +"first_valueMap" +"first_valueMerge" +"first_valueNull" +"first_valueOrDefault" +"first_valueOrNull" +"first_valueResample" +"first_valueSimpleState" +"first_valueState" +"first_value_respect_nulls" +"first_value_respect_nullsArgMax" +"first_value_respect_nullsArgMin" +"first_value_respect_nullsArray" +"first_value_respect_nullsDistinct" +"first_value_respect_nullsForEach" +"first_value_respect_nullsIf" +"first_value_respect_nullsMap" +"first_value_respect_nullsMerge" +"first_value_respect_nullsNull" +"first_value_respect_nullsOrDefault" +"first_value_respect_nullsOrNull" +"first_value_respect_nullsResample" +"first_value_respect_nullsSimpleState" +"first_value_respect_nullsState" +"flameGraph" +"flameGraphArgMax" +"flameGraphArgMin" +"flameGraphArray" +"flameGraphDistinct" +"flameGraphForEach" +"flameGraphIf" +"flameGraphMap" +"flameGraphMerge" +"flameGraphNull" +"flameGraphOrDefault" +"flameGraphOrNull" +"flameGraphResample" +"flameGraphSimpleState" +"flameGraphState" +"flatten" +"flattenTuple" +"floor" +"format" +"formatDateTime" +"formatDateTimeInJodaSyntax" +"formatQuery" +"formatQueryOrNull" +"formatQuerySingleLine" +"formatQuerySingleLineOrNull" +"formatReadableDecimalSize" +"formatReadableQuantity" +"formatReadableSize" +"formatReadableTimeDelta" "formatRow" "formatRowNoNewline" -"dictGetHierarchy" -"notEmpty" -"format" -"murmurHash3_64" -"reverseUTF8" -"arrayEnumerateUniqRanked" -"CRC64" -"pow" -"toInt16" -"reverse" -"lower" -"toStartOfDay" -"caseWithoutExpr" -"CHAR_LENGTH" -"lengthUTF8" -"length" -"bitTestAny" -"floor" -"round" -"reinterpretAsFixedString" -"reinterpretAsString" -"h3ToChildren" -"emptyArrayToSingle" -"tgamma" -"reinterpretAsInt128" -"visitParamExtractRaw" -"reinterpretAsInt64" -"reinterpretAsUInt32" -"reinterpretAsUInt16" -"fuzzBits" -"randomFixedString" -"tupleHammingDistance" -"JSONExtractBool" -"ngramSearchUTF8" -"generateUUIDv4" -"multiSearchAllPositionsCaseInsensitive" -"randConstant" -"leftPad" -"convertCharset" -"extractURLParameter" -"zookeeperSessionUptime" -"serverUUID" -"firstSignificantSubdomain" -"rand64" -"JSON_EXISTS" -"isIPAddressInRange" -"wordShingleSimHashCaseInsensitive" -"parseDateTime32BestEffortOrNull" -"cutToFirstSignificantSubdomainWithWWW" -"roundToExp2" -"mapSubtract" -"partitionId" -"intExp2" -"byteSize" -"errorCodeToName" -"toDateTime64" -"yesterday" -"initializeAggregation" -"globalVariable" -"countDigits" -"toStartOfHour" -"toWeek" -"isDecimalOverflow" -"visitParamExtractString" -"isConstant" -"getSetting" -"shardNum" -"__getScalar" -"position" -"__bitBoolMaskOr" -"cosh" -"basename" -"evalMLMethod" -"filesystemUnreserved" -"filesystemCapacity" -"reinterpretAsDate" -"filesystemAvailable" -"joinGet" -"globalNullInIgnoreSet" -"globalNotInIgnoreSet" -"toRelativeDayNum" -"splitByChar" -"notInIgnoreSet" -"globalInIgnoreSet" -"notNullIn" -"ngramSearchCaseInsensitive" -"notIn" -"trunc" -"JSONExtractArrayRaw" -"version" -"joinGetOrNull" -"globalIn" -"toDate32" -"lowCardinalityIndices" -"toLowCardinality" -"runningConcurrency" -"runningDifferenceStartingWithFirstValue" -"bitCount" -"timezoneOf" -"timezone" -"uptime" -"pi" -"extractTextFromHTML" -"UUIDStringToNum" -"roundDown" -"IPv6NumToString" -"throwIf" -"ifNotFinite" -"FROM_UNIXTIME" -"isInfinite" -"arrayProduct" -"polygonsIntersectionSpherical" -"toRelativeHourNum" -"hasColumnInTable" -"bar" -"JSONExtractKeysAndValues" -"replicate" -"arrayJoin" -"s2RectUnion" -"ngramDistanceUTF8" -"identity" -"indexHint" -"ignore" -"regionHierarchy" -"toDateTime64OrNull" -"polygonsWithinCartesian" -"materialize" -"sleepEachRow" -"parseDateTimeBestEffortUSOrNull" -"neighbor" -"splitByString" -"rowNumberInBlock" -"blockSize" -"defaultValueOfTypeName" -"hiveHash" -"randomStringUTF8" -"blockSerializedSize" -"visibleWidth" -"fullHostName" -"hostName" -"arrayFilter" -"defaultRoles" -"currentRoles" -"toUInt64OrZero" -"dictGetUInt16" -"DATABASE" -"toUInt8OrZero" -"reinterpretAsInt8" -"not" -"log1p" -"and" -"wordShingleMinHash" -"arrayDifference" -"arrayCumSumNonNegative" -"wordShingleSimHash" -"arrayCumSum" -"arraySort" -"arrayPartialSort" -"arrayPartialReverseSort" -"toDaysSinceYearZero" +"fragment" "fromDaysSinceYearZero" "fromDaysSinceYearZero32" -"dumpColumnStructure" -"multiSearchFirstIndex" -"arrayReverseSplit" -"ifNull" -"normalizeQueryKeepNames" -"xxHash64" -"in" -"arrayReverseFill" -"erfc" -"cutToFirstSignificantSubdomainCustomWithWWW" -"arrayFill" -"substringUTF8" -"arrayFirst" -"arrayAvg" -"cos" -"arrayMax" -"toColumnTypeName" -"arrayMin" -"toIntervalDay" -"reinterpretAsFloat32" -"arrayAll" -"or" -"arrayExists" -"trimLeft" -"arrayMap" -"polygonsSymDifferenceCartesian" -"gccMurmurHash" -"exp2" -"polygonPerimeterCartesian" -"SHA384" -"murmurHash3_128" -"polygonConvexHullCartesian" -"murmurHash3_32" -"h3GetBaseCell" -"murmurHash2_64" -"globalNullIn" -"javaHashUTF16LE" -"URLHash" -"reinterpretAsInt16" -"regionToCountry" -"toUUID" -"intHash32" -"metroHash64" -"arrayPushFront" -"bitmapAndnotCardinality" -"toFloat32" -"sipHash64" -"JSONExtractFloat" -"SHA512" -"JSONExtractUInt" -"arrayReduce" -"SHA256" -"array" -"parseDateTimeBestEffortUSOrZero" -"toInt32OrNull" -"MD5" -"randomString" -"__bitBoolMaskAnd" -"hasSubstr" -"formatReadableQuantity" -"replaceAll" -"toIntervalWeek" -"toRelativeSecondNum" -"demangle" -"toNullable" -"concat" -"formatReadableDecimalSize" -"formatReadableSize" -"shardCount" +"fromModifiedJulianDay" "fromModifiedJulianDayOrNull" -"toInt128OrZero" -"bitmaskToList" -"xor" -"bitPositionsToArray" -"empty" -"ngramMinHashArg" -"_CAST" -"dictGetStringOrDefault" -"arrayZip" -"dictGetUUIDOrDefault" -"dictGetFloat64OrDefault" -"dictGetInt64OrDefault" -"today" -"dictGetUInt32OrDefault" -"countSubstringsCaseInsensitiveUTF8" -"h3IsPentagon" -"dictGetUInt16OrDefault" -"dictGetDescendants" -"dictGetString" -"inIgnoreSet" -"dictGetUUID" -"arraySlice" -"reinterpretAsInt32" -"dictGetFloat64" -"globalNotNullInIgnoreSet" -"arrayUniq" -"polygonAreaCartesian" -"intDiv" -"cityHash64" -"regionToTopContinent" -"toInt32" -"e" -"wordShingleSimHashCaseInsensitiveUTF8" -"JSON_QUERY" -"isIPv4String" -"UUIDNumToString" -"modulo" -"reinterpret" -"unbin" -"regionToDistrict" -"sinh" -"date_trunc" -"__bitSwapLastTwo" -"formatDateTime" -"dateDiff" -"tanh" -"multiSearchAnyCaseInsensitiveUTF8" -"upperUTF8" -"bitmaskToArray" -"arrayAUC" -"multiMatchAllIndices" -"hasThreadFuzzer" -"toDayOfWeek" -"isFinite" -"transform" -"timezoneOffset" -"toTimezone" -"cutQueryString" -"subtractYears" -"tcpPort" -"subtractHours" -"tid" -"toDateTime64OrZero" -"subtractSeconds" -"alphaTokens" -"negate" -"file" -"roundAge" -"MACStringToOUI" -"addQuarters" -"arrayResize" -"addWeeks" -"HOUR" -"addHours" -"visitParamExtractUInt" -"caseWithExpression" -"rand" -"addSeconds" -"splitByNonAlpha" -"arrayStringConcat" -"randomPrintableASCII" -"h3HexAreaM2" -"toYYYYMMDDhhmmss" -"toInt16OrNull" -"timeSlots" -"timeSlot" -"dictGetInt16" -"queryString" -"javaHash" -"acosh" -"toRelativeMinuteNum" -"toRelativeYearNum" -"unhex" -"toInt8OrNull" -"regionToPopulation" -"toStartOfISOYear" -"getSizeOfEnumType" -"toStartOfInterval" -"toYYYYMMDD" -"multiSearchFirstPositionUTF8" -"CRC32" -"toStartOfFifteenMinutes" -"emptyArrayUInt8" -"dictGetUInt8" -"toStartOfFiveMinutes" -"cbrt" -"toStartOfMinute" -"dictGet" -"toStartOfSecond" -"hasTokenCaseInsensitive" -"toStartOfYear" -"farmHash64" -"toStartOfMonth" -"queryID" -"wordShingleMinHashCaseInsensitive" -"dictGetDateOrDefault" -"dictGetUInt64OrDefault" -"arrayElement" -"sleep" -"dictHas" -"enabledProfiles" -"toDecimal32OrNull" -"multiFuzzyMatchAny" -"toStartOfWeek" -"isValidUTF8" -"dictGetInt8" -"dictGetInt32OrDefault" -"toMonday" -"multiSearchFirstIndexUTF8" -"lowCardinalityKeys" -"halfMD5" -"domainWithoutWWW" -"MINUTE" -"addDays" -"toMinute" -"JSONExtractRaw" -"toUInt16OrNull" -"bin" -"toDayOfYear" -"log" -"DAYOFMONTH" -"toUInt32" -"toDecimal32" -"s2RectAdd" -"dictGetFloat32OrDefault" -"xxHash32" -"DAY" -"toISOWeek" -"ngramMinHash" -"toMonth" -"countMatches" -"toQuarter" -"toISOYear" -"YEAR" -"toIntervalYear" -"hypot" -"multiSearchAny" -"toIntervalQuarter" -"toSecond" -"accurate_CastOrNull" -"reinterpretAsUInt64" -"defaultProfiles" -"toIntervalMonth" -"bitRotateRight" -"arrayReverseSort" -"toInt8OrZero" -"SHA1" -"multiSearchFirstPositionCaseInsensitiveUTF8" -"cutFragment" -"toDecimal256" -"toIntervalSecond" -"toUInt256OrZero" -"toStringCutToZero" -"normalizedQueryHashKeepNames" -"coalesce" -"parseDateTime64BestEffortOrNull" -"toIntervalHour" -"bitmapSubsetLimit" -"parseDateTimeBestEffortOrNull" -"match" -"fragment" -"repeat" -"parseDateTimeBestEffortOrZero" -"toDecimal128OrNull" -"parseDateTimeBestEffort" -"simpleJSONExtractUInt" -"multiMatchAny" -"toDecimal256OrNull" -"roundDuration" -"addMinutes" -"toStartOfQuarter" -"exp10" -"regionIn" -"toDecimal64OrNull" -"toDateTimeOrNull" -"simpleJSONExtractFloat" -"reinterpretAsUUID" -"dictGetDateTimeOrDefault" -"greater" -"toRelativeMonthNum" -"toDate32OrNull" -"h3IndexesAreNeighbors" -"toFloat32OrNull" -"toInt256OrNull" -"leftPadUTF8" -"regionToArea" -"toInt64OrNull" -"toModifiedJulianDayOrNull" -"toUInt64OrNull" -"polygonsEqualsCartesian" -"accurateCast" -"toUInt32OrNull" -"encodeXMLComponent" -"toUInt8OrNull" -"toUUIDOrZero" -"arrayEnumerateDenseRanked" -"toYYYYMM" -"toDecimal256OrZero" -"fromUnixTimestamp64Milli" -"CHARACTER_LENGTH" -"isNull" -"isNaN" -"multiSearchAllPositionsCaseInsensitiveUTF8" -"FQDN" -"currentProfiles" -"toDateTime32" -"arraySplit" -"toIPv6" -"toDecimal32OrZero" -"bitmapCardinality" -"arrayFlatten" -"dateName" -"s2RectIntersection" -"CAST" -"caseWithExpr" -"nullIf" -"toDateTimeOrZero" -"bitOr" -"CRC32IEEE" -"emptyArrayDate" -"toUInt128OrZero" -"toUInt128OrNull" -"toFloat32OrZero" -"SECOND" -"arrayCompact" -"bitmapMin" -"toInt16OrZero" -"reinterpretAsUInt256" -"dictGetOrNull" -"map" -"bitmapHasAny" -"emptyArrayUInt16" -"cutIPv6" -"toUInt32OrZero" -"toTypeName" -"polygonsUnionCartesian" -"char" -"mapContains" -"emptyArrayDateTime" -"toUInt16OrZero" -"toInt64" -"wkt" -"dictGetUInt64" -"JSONHas" -"runningDifference" -"getServerPort" -"IPv4ToIPv6" -"__bitWrapperFunc" -"multiply" -"accurate_Cast" -"toDecimal64" -"toString" -"arrayIntersect" -"dictGetUInt8OrDefault" -"toTime" -"hasAll" -"now" -"parseDateTime64BestEffortOrZero" -"rightPadUTF8" -"emptyArrayInt64" -"simpleJSONExtractInt" -"toDecimal128" -"toRelativeWeekNum" -"toDateOrNull" -"topLevelDomain" -"greatest" -"isIPv6String" -"toHour" -"toFixedString" -"bitmapOrCardinality" -"toInt128" -"toDateOrZero" -"emptyArrayUInt64" -"toUInt128" -"isNotNull" -"toUInt16" -"MACNumToString" -"multiSearchAllPositionsUTF8" -"toUInt8" -"regionToCity" -"DAYOFYEAR" -"simpleJSONExtractRaw" -"appendTrailingCharIfAbsent" -"toUInt64" -"regionToName" -"IPv4NumToStringClassC" -"upper" -"greaterOrEquals" -"s2RectContains" -"toDate" -"regexpQuoteMeta" -"readWKTMultiPolygon" -"emptyArrayString" -"bitmapOr" -"cutWWW" -"emptyArrayInt8" -"less" -"readWKTPoint" -"reinterpretAsDateTime" -"notEquals" -"geoToS2" -"nullInIgnoreSet" -"globalNotNullIn" -"arrayConcat" -"mapPopulateSeries" -"parseDateTimeBestEffortUS" -"toDayOfMonth" -"gcd" -"protocol" -"currentUser" -"if" -"caseWithoutExpression" -"currentDatabase" -"log10" -"moduloOrZero" -"MD4" -"toInt64OrZero" -"arrayWithConstant" -"IPv4NumToString" -"dictGetUInt32" -"multiSearchAnyCaseInsensitive" -"countSubstrings" -"reinterpretAsFloat64" -"plus" -"dictGetChildren" -"IPv6CIDRToRange" -"toIntervalMinute" -"bitRotateLeft" -"geoDistance" -"mapValues" -"toIPv4" -"extractGroups" -"toYearWeek" -"polygonsDistanceCartesian" -"lowerUTF8" -"toUInt256OrNull" -"IPv4CIDRToRange" -"MACStringToNum" -"toFloat64OrNull" -"sipHash128" -"toFloat64" -"arrayPopFront" -"bitmapAnd" -"bitmapHasAll" -"subtractWeeks" -"arrayFirstIndex" -"bitmapXorCardinality" -"bitmapAndnot" -"asinh" -"dictGetInt32" -"toFloat64OrZero" -"toInt8" -"polygonsWithinSpherical" -"bitmapXor" -"intExp10" -"dictGetDateTime" -"h3ToGeoBoundary" -"decodeURLComponent" -"bitAnd" -"toUUIDOrNull" -"notNullInIgnoreSet" -"tupleElement" -"tupleToNameValuePairs" -"tuple" -"ceil" -"DAYOFWEEK" -"toDateTime" -"bitNot" -"rightPad" -"has" -"bitmapToArray" -"arrayCount" -"arrayDistinct" -"greatCircleAngle" -"IPv6StringToNum" -"dictGetInt64" -"toUnixTimestamp" -"getMacro" -"runningAccumulate" -"multiIf" -"abs" -"toValidUTF8" -"arrayEnumerateUniq" -"moduloLegacy" -"JSONExtractInt" -"subtractMonths" -"dateTimeToSnowflake" -"cutToFirstSignificantSubdomain" -"roundBankers" -"toUInt256" -"arrayEnumerateDense" -"stem" -"bitmapAndCardinality" -"bitTestAll" -"normalizedQueryHash" -"toYear" -"emptyArrayFloat64" -"bitmapTransform" -"JSONExtract" -"bitShiftRight" -"bitmapContains" -"sqrt" -"minus" -"QUARTER" -"parseDateTime64BestEffort" -"h3IsValid" -"h3EdgeAngle" -"concatAssumeInjective" -"emptyArrayUInt32" -"intHash64" -"enabledRoles" -"fromUnixTimestamp64Micro" -"range" -"subtractMinutes" -"farmFingerprint64" -"arrayReduceInRanges" -"bitmapBuild" -"bitmapSubsetInRange" -"arrayPopBack" -"h3ToGeo" -"hex" -"arrayPushBack" -"subtractDays" -"buildId" -"URLPathHierarchy" -"addMonths" -"parseDateTime32BestEffort" -"indexOf" -"polygonsUnionSpherical" -"arraySum" -"dictGetFloat32" -"ngramSimHashCaseInsensitiveUTF8" -"finalizeAggregation" -"divide" -"netloc" -"bitShiftLeft" -"toInt32OrZero" -"extractAll" -"toInt128OrNull" -"bitXor" -"lcm" -"accurateCastOrNull" -"toDate32OrZero" -"simpleJSONExtractString" -"least" -"nullIn" -"MONTH" -"arrayReverse" -"arrayShuffle" -"arrayPartialShuffle" -"now64" -"DATE" -"addressToLine" -"globalNotIn" -"dateTime64ToSnowflake" -"dictGetInt8OrDefault" -"exp" -"toRelativeQuarterNum" -"arrayEnumerate" -"mapAdd" -"murmurHash2_32" -"toStartOfTenMinutes" -"intDivOrZero" -"addYears" -"queryStringAndFragment" -"emptyArrayFloat32" -"countEqual" -"pointInEllipses" -"like" -"toDecimal64OrZero" -"reinterpretAsInt256" -"hasAny" -"ln" -"replace" -"rpad" -"mid" -"lcase" +"fromUTCTimestamp" "fromUnixTimestamp" -"truncate" -"ceiling" -"ucase" -"rand32" -"power" -"initial_query_id" -"timeZone" -"hostname" -"yearweek" -"timeZoneOffset" -"extractAllGroups" -"lpad" -"substr" -"INET6_NTOA" -"INET6_ATON" -"INET_ATON" -"timeZoneOf" -"locate" -"INET_NTOA" -"connection_id" -"user" -"toTimeZone" -"week" -"flatten" -"query_id" -"dateTrunc" -"mod" -"lagInFrame" -"dense_rank" -"rank" -"exponentialMovingAverage" -"sparkbar" -"singleValueOrNull" -"studentTTest" -"aggThrow" -"categoricalInformationValue" -"groupArrayMovingAvg" -"groupArrayMovingSum" -"simpleLinearRegression" -"entropy" -"quantilesBFloat16" -"maxIntersectionsPosition" -"groupBitmapXor" -"groupBitmap" -"skewPop" -"groupBitXor" -"groupBitOr" -"groupBitmapAnd" -"topKWeighted" -"stochasticLinearRegression" -"corr" -"corrMatrix" -"uniqCombined64" -"intervalLengthSum" -"uniqCombined" -"rankCorr" -"maxMap" -"minMap" -"sumMapFiltered" -"quantileExactWeighted" -"sumMapFilteredWithOverflow" -"sumMap" -"histogram" -"quantiles" -"sum" -"covarPop" -"covarPopMatrix" -"row_number" -"kurtPop" -"kurtSamp" -"skewSamp" -"uniqExact" -"sumMapWithOverflow" -"stddevSamp" -"varPop" -"corrStable" -"quantileTimingWeighted" -"covarPopStable" -"stddevSampStable" -"varSamp" -"topK" -"last_value" -"mannWhitneyUTest" -"maxIntersections" -"quantilesExact" -"uniqHLL12" -"quantileBFloat16" -"uniq" -"min" -"sequenceNextNode" -"quantilesTimingWeighted" -"boundingRatio" -"any" -"anyLast" -"deltaSum" -"retention" -"sequenceMatch" -"uniqUpTo" -"windowFunnel" -"deltaSumTimestamp" -"varSampStable" -"uniqTheta" -"quantilesExactWeighted" -"max" -"quantilesBFloat16Weighted" -"quantileBFloat16Weighted" -"sumKahan" -"quantilesTDigestWeighted" -"groupBitAnd" -"quantileTDigest" -"quantileTDigestWeighted" -"argMax" -"quantileDeterministic" -"quantilesTDigest" -"stochasticLogisticRegression" -"argMin" -"avg" -"covarSampStable" -"quantilesTiming" -"welchTTest" -"covarSamp" -"covarSampMatrix" -"varPopStable" -"quantileTiming" -"quantileExactInclusive" -"quantileExactHigh" -"groupArraySample" -"quantilesExactLow" -"groupBitmapOr" -"first_value" -"quantileExactExclusive" -"quantileExact" -"sumCount" -"groupArrayInsertAt" -"quantilesExactHigh" -"sumWithOverflow" -"sequenceCount" -"quantilesDeterministic" -"groupUniqArray" +"fromUnixTimestamp64Micro" +"fromUnixTimestamp64Milli" +"fromUnixTimestamp64Nano" +"fromUnixTimestampInJodaSyntax" +"from_utc_timestamp" +"fullHostName" +"fuzzBits" +"gccMurmurHash" +"gcd" +"generateRandomStructure" +"generateULID" +"generateUUIDv4" +"geoDistance" +"geoToH3" +"geoToS2" +"geohashDecode" +"geohashEncode" +"geohashesInBox" +"getMacro" +"getOSKernelVersion" +"getServerPort" +"getSetting" +"getSizeOfEnumType" +"getSubcolumn" +"getTypeSerializationStreams" +"globalIn" +"globalInIgnoreSet" +"globalNotIn" +"globalNotInIgnoreSet" +"globalNotNullIn" +"globalNotNullInIgnoreSet" +"globalNullIn" +"globalNullInIgnoreSet" +"globalVariable" +"greatCircleAngle" +"greatCircleDistance" +"greater" +"greaterOrEquals" +"greatest" "groupArray" -"anyHeavy" -"stddevPop" -"quantile" -"leadInFrame" -"quantilesExactExclusive" -"count" -"quantilesExactInclusive" -"stddevPopStable" -"quantileExactLow" -"avgWeighted" -"BIT_AND" -"VAR_SAMP" -"COVAR_SAMP" -"VAR_POP" -"medianTDigest" -"medianBFloat16" -"medianTimingWeighted" -"medianTiming" -"medianExactHigh" -"BIT_OR" -"medianDeterministic" -"STDDEV_POP" -"STDDEV_SAMP" -"medianExactLow" -"medianTDigestWeighted" -"medianExact" -"COVAR_POP" -"medianBFloat16Weighted" -"medianExactWeighted" -"BIT_XOR" -"median" -"log" -"replaceAll" -"rightPad" -"substring" -"lower" -"FROM_UNIXTIME" -"trunc" -"ceil" -"upper" -"rand" -"pow" -"initialQueryID" -"timezone" -"hostName" -"toYearWeek" -"timezoneOffset" -"extractAllGroupsVertical" -"leftPad" -"substring" -"IPv6NumToString" -"IPv6StringToNum" -"IPv4StringToNum" -"timezoneOf" -"position" -"IPv4NumToString" -"connectionid" -"currentUser" -"toTimezone" -"toWeek" -"arrayFlatten" -"queryID" -"date_trunc" -"modulo" +"groupArrayArgMax" +"groupArrayArgMin" +"groupArrayArray" +"groupArrayDistinct" +"groupArrayForEach" +"groupArrayIf" +"groupArrayInsertAt" +"groupArrayInsertAtArgMax" +"groupArrayInsertAtArgMin" +"groupArrayInsertAtArray" +"groupArrayInsertAtDistinct" +"groupArrayInsertAtForEach" +"groupArrayInsertAtIf" +"groupArrayInsertAtMap" +"groupArrayInsertAtMerge" +"groupArrayInsertAtNull" +"groupArrayInsertAtOrDefault" +"groupArrayInsertAtOrNull" +"groupArrayInsertAtResample" +"groupArrayInsertAtSimpleState" +"groupArrayInsertAtState" +"groupArrayIntersect" +"groupArrayIntersectArgMax" +"groupArrayIntersectArgMin" +"groupArrayIntersectArray" +"groupArrayIntersectDistinct" +"groupArrayIntersectForEach" +"groupArrayIntersectIf" +"groupArrayIntersectMap" +"groupArrayIntersectMerge" +"groupArrayIntersectNull" +"groupArrayIntersectOrDefault" +"groupArrayIntersectOrNull" +"groupArrayIntersectResample" +"groupArrayIntersectSimpleState" +"groupArrayIntersectState" +"groupArrayLast" +"groupArrayLastArgMax" +"groupArrayLastArgMin" +"groupArrayLastArray" +"groupArrayLastDistinct" +"groupArrayLastForEach" +"groupArrayLastIf" +"groupArrayLastMap" +"groupArrayLastMerge" +"groupArrayLastNull" +"groupArrayLastOrDefault" +"groupArrayLastOrNull" +"groupArrayLastResample" +"groupArrayLastSimpleState" +"groupArrayLastState" +"groupArrayMap" +"groupArrayMerge" +"groupArrayMovingAvg" +"groupArrayMovingAvgArgMax" +"groupArrayMovingAvgArgMin" +"groupArrayMovingAvgArray" +"groupArrayMovingAvgDistinct" +"groupArrayMovingAvgForEach" +"groupArrayMovingAvgIf" +"groupArrayMovingAvgMap" +"groupArrayMovingAvgMerge" +"groupArrayMovingAvgNull" +"groupArrayMovingAvgOrDefault" +"groupArrayMovingAvgOrNull" +"groupArrayMovingAvgResample" +"groupArrayMovingAvgSimpleState" +"groupArrayMovingAvgState" +"groupArrayMovingSum" +"groupArrayMovingSumArgMax" +"groupArrayMovingSumArgMin" +"groupArrayMovingSumArray" +"groupArrayMovingSumDistinct" +"groupArrayMovingSumForEach" +"groupArrayMovingSumIf" +"groupArrayMovingSumMap" +"groupArrayMovingSumMerge" +"groupArrayMovingSumNull" +"groupArrayMovingSumOrDefault" +"groupArrayMovingSumOrNull" +"groupArrayMovingSumResample" +"groupArrayMovingSumSimpleState" +"groupArrayMovingSumState" +"groupArrayNull" +"groupArrayOrDefault" +"groupArrayOrNull" +"groupArrayResample" +"groupArraySample" +"groupArraySampleArgMax" +"groupArraySampleArgMin" +"groupArraySampleArray" +"groupArraySampleDistinct" +"groupArraySampleForEach" +"groupArraySampleIf" +"groupArraySampleMap" +"groupArraySampleMerge" +"groupArraySampleNull" +"groupArraySampleOrDefault" +"groupArraySampleOrNull" +"groupArraySampleResample" +"groupArraySampleSimpleState" +"groupArraySampleState" +"groupArraySimpleState" +"groupArraySorted" +"groupArraySortedArgMax" +"groupArraySortedArgMin" +"groupArraySortedArray" +"groupArraySortedDistinct" +"groupArraySortedForEach" +"groupArraySortedIf" +"groupArraySortedMap" +"groupArraySortedMerge" +"groupArraySortedNull" +"groupArraySortedOrDefault" +"groupArraySortedOrNull" +"groupArraySortedResample" +"groupArraySortedSimpleState" +"groupArraySortedState" +"groupArrayState" "groupBitAnd" -"varSamp" -"covarSamp" -"varPop" -"quantileTDigest" -"quantileBFloat16" -"quantileTimingWeighted" -"quantileTiming" -"quantileExactHigh" +"groupBitAndArgMax" +"groupBitAndArgMin" +"groupBitAndArray" +"groupBitAndDistinct" +"groupBitAndForEach" +"groupBitAndIf" +"groupBitAndMap" +"groupBitAndMerge" +"groupBitAndNull" +"groupBitAndOrDefault" +"groupBitAndOrNull" +"groupBitAndResample" +"groupBitAndSimpleState" +"groupBitAndState" "groupBitOr" -"quantileDeterministic" -"stddevPop" -"stddevSamp" -"quantileExactLow" -"quantileTDigestWeighted" -"quantileExact" -"covarPop" -"quantileBFloat16Weighted" -"quantileExactWeighted" +"groupBitOrArgMax" +"groupBitOrArgMin" +"groupBitOrArray" +"groupBitOrDistinct" +"groupBitOrForEach" +"groupBitOrIf" +"groupBitOrMap" +"groupBitOrMerge" +"groupBitOrNull" +"groupBitOrOrDefault" +"groupBitOrOrNull" +"groupBitOrResample" +"groupBitOrSimpleState" +"groupBitOrState" "groupBitXor" +"groupBitXorArgMax" +"groupBitXorArgMin" +"groupBitXorArray" +"groupBitXorDistinct" +"groupBitXorForEach" +"groupBitXorIf" +"groupBitXorMap" +"groupBitXorMerge" +"groupBitXorNull" +"groupBitXorOrDefault" +"groupBitXorOrNull" +"groupBitXorResample" +"groupBitXorSimpleState" +"groupBitXorState" +"groupBitmap" +"groupBitmapAnd" +"groupBitmapAndArgMax" +"groupBitmapAndArgMin" +"groupBitmapAndArray" +"groupBitmapAndDistinct" +"groupBitmapAndForEach" +"groupBitmapAndIf" +"groupBitmapAndMap" +"groupBitmapAndMerge" +"groupBitmapAndNull" +"groupBitmapAndOrDefault" +"groupBitmapAndOrNull" +"groupBitmapAndResample" +"groupBitmapAndSimpleState" +"groupBitmapAndState" +"groupBitmapArgMax" +"groupBitmapArgMin" +"groupBitmapArray" +"groupBitmapDistinct" +"groupBitmapForEach" +"groupBitmapIf" +"groupBitmapMap" +"groupBitmapMerge" +"groupBitmapNull" +"groupBitmapOr" +"groupBitmapOrArgMax" +"groupBitmapOrArgMin" +"groupBitmapOrArray" +"groupBitmapOrDefault" +"groupBitmapOrDistinct" +"groupBitmapOrForEach" +"groupBitmapOrIf" +"groupBitmapOrMap" +"groupBitmapOrMerge" +"groupBitmapOrNull" +"groupBitmapOrNull" +"groupBitmapOrOrDefault" +"groupBitmapOrOrNull" +"groupBitmapOrResample" +"groupBitmapOrSimpleState" +"groupBitmapOrState" +"groupBitmapResample" +"groupBitmapSimpleState" +"groupBitmapState" +"groupBitmapXor" +"groupBitmapXorArgMax" +"groupBitmapXorArgMin" +"groupBitmapXorArray" +"groupBitmapXorDistinct" +"groupBitmapXorForEach" +"groupBitmapXorIf" +"groupBitmapXorMap" +"groupBitmapXorMerge" +"groupBitmapXorNull" +"groupBitmapXorOrDefault" +"groupBitmapXorOrNull" +"groupBitmapXorResample" +"groupBitmapXorSimpleState" +"groupBitmapXorState" +"groupUniqArray" +"groupUniqArrayArgMax" +"groupUniqArrayArgMin" +"groupUniqArrayArray" +"groupUniqArrayDistinct" +"groupUniqArrayForEach" +"groupUniqArrayIf" +"groupUniqArrayMap" +"groupUniqArrayMerge" +"groupUniqArrayNull" +"groupUniqArrayOrDefault" +"groupUniqArrayOrNull" +"groupUniqArrayResample" +"groupUniqArraySimpleState" +"groupUniqArrayState" +"h3CellAreaM2" +"h3CellAreaRads2" +"h3Distance" +"h3EdgeAngle" +"h3EdgeLengthKm" +"h3EdgeLengthM" +"h3ExactEdgeLengthKm" +"h3ExactEdgeLengthM" +"h3ExactEdgeLengthRads" +"h3GetBaseCell" +"h3GetDestinationIndexFromUnidirectionalEdge" +"h3GetFaces" +"h3GetIndexesFromUnidirectionalEdge" +"h3GetOriginIndexFromUnidirectionalEdge" +"h3GetPentagonIndexes" +"h3GetRes0Indexes" +"h3GetResolution" +"h3GetUnidirectionalEdge" +"h3GetUnidirectionalEdgeBoundary" +"h3GetUnidirectionalEdgesFromHexagon" +"h3HexAreaKm2" +"h3HexAreaM2" +"h3HexRing" +"h3IndexesAreNeighbors" +"h3IsPentagon" +"h3IsResClassIII" +"h3IsValid" +"h3Line" +"h3NumHexagons" +"h3PointDistKm" +"h3PointDistM" +"h3PointDistRads" +"h3ToCenterChild" +"h3ToChildren" +"h3ToGeo" +"h3ToGeoBoundary" +"h3ToParent" +"h3ToString" +"h3UnidirectionalEdgeIsValid" +"h3kRing" +"halfMD5" +"has" +"hasAll" +"hasAny" +"hasColumnInTable" +"hasSubsequence" +"hasSubsequenceCaseInsensitive" +"hasSubsequenceCaseInsensitiveUTF8" +"hasSubsequenceUTF8" +"hasSubstr" +"hasThreadFuzzer" +"hasToken" +"hasTokenCaseInsensitive" +"hasTokenCaseInsensitiveOrNull" +"hasTokenOrNull" +"hex" +"histogram" +"histogramArgMax" +"histogramArgMin" +"histogramArray" +"histogramDistinct" +"histogramForEach" +"histogramIf" +"histogramMap" +"histogramMerge" +"histogramNull" +"histogramOrDefault" +"histogramOrNull" +"histogramResample" +"histogramSimpleState" +"histogramState" +"hiveHash" +"hop" +"hopEnd" +"hopStart" +"hostName" +"hostname" +"hypot" +"identity" +"idnaDecode" +"idnaEncode" +"if" +"ifNotFinite" +"ifNull" +"ignore" +"ilike" +"in" +"inIgnoreSet" +"indexHint" +"indexOf" +"initcap" +"initcapUTF8" +"initialQueryID" +"initial_query_id" +"initializeAggregation" +"instr" +"intDiv" +"intDivOrZero" +"intExp10" +"intExp2" +"intHash32" +"intHash64" +"intervalLengthSum" +"intervalLengthSumArgMax" +"intervalLengthSumArgMin" +"intervalLengthSumArray" +"intervalLengthSumDistinct" +"intervalLengthSumForEach" +"intervalLengthSumIf" +"intervalLengthSumMap" +"intervalLengthSumMerge" +"intervalLengthSumNull" +"intervalLengthSumOrDefault" +"intervalLengthSumOrNull" +"intervalLengthSumResample" +"intervalLengthSumSimpleState" +"intervalLengthSumState" +"isConstant" +"isDecimalOverflow" +"isFinite" +"isIPAddressInRange" +"isIPv4String" +"isIPv6String" +"isInfinite" +"isNaN" +"isNotDistinctFrom" +"isNotNull" +"isNull" +"isNullable" +"isValidJSON" +"isValidUTF8" +"isZeroOrNull" +"jaroSimilarity" +"jaroWinklerSimilarity" +"javaHash" +"javaHashUTF16LE" +"joinGet" +"joinGetOrNull" +"jsonMergePatch" +"jumpConsistentHash" +"kafkaMurmurHash" +"kolmogorovSmirnovTest" +"kolmogorovSmirnovTestArgMax" +"kolmogorovSmirnovTestArgMin" +"kolmogorovSmirnovTestArray" +"kolmogorovSmirnovTestDistinct" +"kolmogorovSmirnovTestForEach" +"kolmogorovSmirnovTestIf" +"kolmogorovSmirnovTestMap" +"kolmogorovSmirnovTestMerge" +"kolmogorovSmirnovTestNull" +"kolmogorovSmirnovTestOrDefault" +"kolmogorovSmirnovTestOrNull" +"kolmogorovSmirnovTestResample" +"kolmogorovSmirnovTestSimpleState" +"kolmogorovSmirnovTestState" +"kostikConsistentHash" +"kql_array_sort_asc" +"kql_array_sort_desc" +"kurtPop" +"kurtPopArgMax" +"kurtPopArgMin" +"kurtPopArray" +"kurtPopDistinct" +"kurtPopForEach" +"kurtPopIf" +"kurtPopMap" +"kurtPopMerge" +"kurtPopNull" +"kurtPopOrDefault" +"kurtPopOrNull" +"kurtPopResample" +"kurtPopSimpleState" +"kurtPopState" +"kurtSamp" +"kurtSampArgMax" +"kurtSampArgMin" +"kurtSampArray" +"kurtSampDistinct" +"kurtSampForEach" +"kurtSampIf" +"kurtSampMap" +"kurtSampMerge" +"kurtSampNull" +"kurtSampOrDefault" +"kurtSampOrNull" +"kurtSampResample" +"kurtSampSimpleState" +"kurtSampState" +"lagInFrame" +"lagInFrameArgMax" +"lagInFrameArgMin" +"lagInFrameArray" +"lagInFrameDistinct" +"lagInFrameForEach" +"lagInFrameIf" +"lagInFrameMap" +"lagInFrameMerge" +"lagInFrameNull" +"lagInFrameOrDefault" +"lagInFrameOrNull" +"lagInFrameResample" +"lagInFrameSimpleState" +"lagInFrameState" +"largestTriangleThreeBuckets" +"largestTriangleThreeBucketsArgMax" +"largestTriangleThreeBucketsArgMin" +"largestTriangleThreeBucketsArray" +"largestTriangleThreeBucketsDistinct" +"largestTriangleThreeBucketsForEach" +"largestTriangleThreeBucketsIf" +"largestTriangleThreeBucketsMap" +"largestTriangleThreeBucketsMerge" +"largestTriangleThreeBucketsNull" +"largestTriangleThreeBucketsOrDefault" +"largestTriangleThreeBucketsOrNull" +"largestTriangleThreeBucketsResample" +"largestTriangleThreeBucketsSimpleState" +"largestTriangleThreeBucketsState" +"last_value" +"last_valueArgMax" +"last_valueArgMin" +"last_valueArray" +"last_valueDistinct" +"last_valueForEach" +"last_valueIf" +"last_valueMap" +"last_valueMerge" +"last_valueNull" +"last_valueOrDefault" +"last_valueOrNull" +"last_valueResample" +"last_valueSimpleState" +"last_valueState" +"last_value_respect_nulls" +"last_value_respect_nullsArgMax" +"last_value_respect_nullsArgMin" +"last_value_respect_nullsArray" +"last_value_respect_nullsDistinct" +"last_value_respect_nullsForEach" +"last_value_respect_nullsIf" +"last_value_respect_nullsMap" +"last_value_respect_nullsMerge" +"last_value_respect_nullsNull" +"last_value_respect_nullsOrDefault" +"last_value_respect_nullsOrNull" +"last_value_respect_nullsResample" +"last_value_respect_nullsSimpleState" +"last_value_respect_nullsState" +"lcase" +"lcm" +"leadInFrame" +"leadInFrameArgMax" +"leadInFrameArgMin" +"leadInFrameArray" +"leadInFrameDistinct" +"leadInFrameForEach" +"leadInFrameIf" +"leadInFrameMap" +"leadInFrameMerge" +"leadInFrameNull" +"leadInFrameOrDefault" +"leadInFrameOrNull" +"leadInFrameResample" +"leadInFrameSimpleState" +"leadInFrameState" +"least" +"left" +"leftPad" +"leftPadUTF8" +"leftUTF8" +"lemmatize" +"length" +"lengthUTF8" +"less" +"lessOrEquals" +"levenshteinDistance" +"lgamma" +"like" +"ln" +"locate" +"log" +"log10" +"log1p" +"log2" +"logTrace" +"lowCardinalityIndices" +"lowCardinalityKeys" +"lower" +"lowerUTF8" +"lpad" +"ltrim" +"lttb" +"lttbArgMax" +"lttbArgMin" +"lttbArray" +"lttbDistinct" +"lttbForEach" +"lttbIf" +"lttbMap" +"lttbMerge" +"lttbNull" +"lttbOrDefault" +"lttbOrNull" +"lttbResample" +"lttbSimpleState" +"lttbState" +"makeDate" +"makeDate32" +"makeDateTime" +"makeDateTime64" +"mannWhitneyUTest" +"mannWhitneyUTestArgMax" +"mannWhitneyUTestArgMin" +"mannWhitneyUTestArray" +"mannWhitneyUTestDistinct" +"mannWhitneyUTestForEach" +"mannWhitneyUTestIf" +"mannWhitneyUTestMap" +"mannWhitneyUTestMerge" +"mannWhitneyUTestNull" +"mannWhitneyUTestOrDefault" +"mannWhitneyUTestOrNull" +"mannWhitneyUTestResample" +"mannWhitneyUTestSimpleState" +"mannWhitneyUTestState" +"map" +"mapAdd" +"mapAll" +"mapApply" +"mapConcat" +"mapContains" +"mapContainsKeyLike" +"mapExists" +"mapExtractKeyLike" +"mapFilter" +"mapFromArrays" +"mapFromString" +"mapKeys" +"mapPartialReverseSort" +"mapPartialSort" +"mapPopulateSeries" +"mapReverseSort" +"mapSort" +"mapSubtract" +"mapUpdate" +"mapValues" +"match" +"materialize" +"max" +"max2" +"maxArgMax" +"maxArgMin" +"maxArray" +"maxDistinct" +"maxForEach" +"maxIf" +"maxIntersections" +"maxIntersectionsArgMax" +"maxIntersectionsArgMin" +"maxIntersectionsArray" +"maxIntersectionsDistinct" +"maxIntersectionsForEach" +"maxIntersectionsIf" +"maxIntersectionsMap" +"maxIntersectionsMerge" +"maxIntersectionsNull" +"maxIntersectionsOrDefault" +"maxIntersectionsOrNull" +"maxIntersectionsPosition" +"maxIntersectionsPositionArgMax" +"maxIntersectionsPositionArgMin" +"maxIntersectionsPositionArray" +"maxIntersectionsPositionDistinct" +"maxIntersectionsPositionForEach" +"maxIntersectionsPositionIf" +"maxIntersectionsPositionMap" +"maxIntersectionsPositionMerge" +"maxIntersectionsPositionNull" +"maxIntersectionsPositionOrDefault" +"maxIntersectionsPositionOrNull" +"maxIntersectionsPositionResample" +"maxIntersectionsPositionSimpleState" +"maxIntersectionsPositionState" +"maxIntersectionsResample" +"maxIntersectionsSimpleState" +"maxIntersectionsState" +"maxMap" +"maxMappedArrays" +"maxMappedArraysArgMax" +"maxMappedArraysArgMin" +"maxMappedArraysArray" +"maxMappedArraysDistinct" +"maxMappedArraysForEach" +"maxMappedArraysIf" +"maxMappedArraysMap" +"maxMappedArraysMerge" +"maxMappedArraysNull" +"maxMappedArraysOrDefault" +"maxMappedArraysOrNull" +"maxMappedArraysResample" +"maxMappedArraysSimpleState" +"maxMappedArraysState" +"maxMerge" +"maxNull" +"maxOrDefault" +"maxOrNull" +"maxResample" +"maxSimpleState" +"maxState" +"meanZTest" +"meanZTestArgMax" +"meanZTestArgMin" +"meanZTestArray" +"meanZTestDistinct" +"meanZTestForEach" +"meanZTestIf" +"meanZTestMap" +"meanZTestMerge" +"meanZTestNull" +"meanZTestOrDefault" +"meanZTestOrNull" +"meanZTestResample" +"meanZTestSimpleState" +"meanZTestState" +"median" +"medianArgMax" +"medianArgMin" +"medianArray" +"medianBFloat16" +"medianBFloat16ArgMax" +"medianBFloat16ArgMin" +"medianBFloat16Array" +"medianBFloat16Distinct" +"medianBFloat16ForEach" +"medianBFloat16If" +"medianBFloat16Map" +"medianBFloat16Merge" +"medianBFloat16Null" +"medianBFloat16OrDefault" +"medianBFloat16OrNull" +"medianBFloat16Resample" +"medianBFloat16SimpleState" +"medianBFloat16State" +"medianBFloat16Weighted" +"medianBFloat16WeightedArgMax" +"medianBFloat16WeightedArgMin" +"medianBFloat16WeightedArray" +"medianBFloat16WeightedDistinct" +"medianBFloat16WeightedForEach" +"medianBFloat16WeightedIf" +"medianBFloat16WeightedMap" +"medianBFloat16WeightedMerge" +"medianBFloat16WeightedNull" +"medianBFloat16WeightedOrDefault" +"medianBFloat16WeightedOrNull" +"medianBFloat16WeightedResample" +"medianBFloat16WeightedSimpleState" +"medianBFloat16WeightedState" +"medianDD" +"medianDDArgMax" +"medianDDArgMin" +"medianDDArray" +"medianDDDistinct" +"medianDDForEach" +"medianDDIf" +"medianDDMap" +"medianDDMerge" +"medianDDNull" +"medianDDOrDefault" +"medianDDOrNull" +"medianDDResample" +"medianDDSimpleState" +"medianDDState" +"medianDeterministic" +"medianDeterministicArgMax" +"medianDeterministicArgMin" +"medianDeterministicArray" +"medianDeterministicDistinct" +"medianDeterministicForEach" +"medianDeterministicIf" +"medianDeterministicMap" +"medianDeterministicMerge" +"medianDeterministicNull" +"medianDeterministicOrDefault" +"medianDeterministicOrNull" +"medianDeterministicResample" +"medianDeterministicSimpleState" +"medianDeterministicState" +"medianDistinct" +"medianExact" +"medianExactArgMax" +"medianExactArgMin" +"medianExactArray" +"medianExactDistinct" +"medianExactForEach" +"medianExactHigh" +"medianExactHighArgMax" +"medianExactHighArgMin" +"medianExactHighArray" +"medianExactHighDistinct" +"medianExactHighForEach" +"medianExactHighIf" +"medianExactHighMap" +"medianExactHighMerge" +"medianExactHighNull" +"medianExactHighOrDefault" +"medianExactHighOrNull" +"medianExactHighResample" +"medianExactHighSimpleState" +"medianExactHighState" +"medianExactIf" +"medianExactLow" +"medianExactLowArgMax" +"medianExactLowArgMin" +"medianExactLowArray" +"medianExactLowDistinct" +"medianExactLowForEach" +"medianExactLowIf" +"medianExactLowMap" +"medianExactLowMerge" +"medianExactLowNull" +"medianExactLowOrDefault" +"medianExactLowOrNull" +"medianExactLowResample" +"medianExactLowSimpleState" +"medianExactLowState" +"medianExactMap" +"medianExactMerge" +"medianExactNull" +"medianExactOrDefault" +"medianExactOrNull" +"medianExactResample" +"medianExactSimpleState" +"medianExactState" +"medianExactWeighted" +"medianExactWeightedArgMax" +"medianExactWeightedArgMin" +"medianExactWeightedArray" +"medianExactWeightedDistinct" +"medianExactWeightedForEach" +"medianExactWeightedIf" +"medianExactWeightedMap" +"medianExactWeightedMerge" +"medianExactWeightedNull" +"medianExactWeightedOrDefault" +"medianExactWeightedOrNull" +"medianExactWeightedResample" +"medianExactWeightedSimpleState" +"medianExactWeightedState" +"medianForEach" +"medianGK" +"medianGKArgMax" +"medianGKArgMin" +"medianGKArray" +"medianGKDistinct" +"medianGKForEach" +"medianGKIf" +"medianGKMap" +"medianGKMerge" +"medianGKNull" +"medianGKOrDefault" +"medianGKOrNull" +"medianGKResample" +"medianGKSimpleState" +"medianGKState" +"medianIf" +"medianInterpolatedWeighted" +"medianInterpolatedWeightedArgMax" +"medianInterpolatedWeightedArgMin" +"medianInterpolatedWeightedArray" +"medianInterpolatedWeightedDistinct" +"medianInterpolatedWeightedForEach" +"medianInterpolatedWeightedIf" +"medianInterpolatedWeightedMap" +"medianInterpolatedWeightedMerge" +"medianInterpolatedWeightedNull" +"medianInterpolatedWeightedOrDefault" +"medianInterpolatedWeightedOrNull" +"medianInterpolatedWeightedResample" +"medianInterpolatedWeightedSimpleState" +"medianInterpolatedWeightedState" +"medianMap" +"medianMerge" +"medianNull" +"medianOrDefault" +"medianOrNull" +"medianResample" +"medianSimpleState" +"medianState" +"medianTDigest" +"medianTDigestArgMax" +"medianTDigestArgMin" +"medianTDigestArray" +"medianTDigestDistinct" +"medianTDigestForEach" +"medianTDigestIf" +"medianTDigestMap" +"medianTDigestMerge" +"medianTDigestNull" +"medianTDigestOrDefault" +"medianTDigestOrNull" +"medianTDigestResample" +"medianTDigestSimpleState" +"medianTDigestState" +"medianTDigestWeighted" +"medianTDigestWeightedArgMax" +"medianTDigestWeightedArgMin" +"medianTDigestWeightedArray" +"medianTDigestWeightedDistinct" +"medianTDigestWeightedForEach" +"medianTDigestWeightedIf" +"medianTDigestWeightedMap" +"medianTDigestWeightedMerge" +"medianTDigestWeightedNull" +"medianTDigestWeightedOrDefault" +"medianTDigestWeightedOrNull" +"medianTDigestWeightedResample" +"medianTDigestWeightedSimpleState" +"medianTDigestWeightedState" +"medianTiming" +"medianTimingArgMax" +"medianTimingArgMin" +"medianTimingArray" +"medianTimingDistinct" +"medianTimingForEach" +"medianTimingIf" +"medianTimingMap" +"medianTimingMerge" +"medianTimingNull" +"medianTimingOrDefault" +"medianTimingOrNull" +"medianTimingResample" +"medianTimingSimpleState" +"medianTimingState" +"medianTimingWeighted" +"medianTimingWeightedArgMax" +"medianTimingWeightedArgMin" +"medianTimingWeightedArray" +"medianTimingWeightedDistinct" +"medianTimingWeightedForEach" +"medianTimingWeightedIf" +"medianTimingWeightedMap" +"medianTimingWeightedMerge" +"medianTimingWeightedNull" +"medianTimingWeightedOrDefault" +"medianTimingWeightedOrNull" +"medianTimingWeightedResample" +"medianTimingWeightedSimpleState" +"medianTimingWeightedState" +"metroHash64" +"mid" +"min" +"min2" +"minArgMax" +"minArgMin" +"minArray" +"minDistinct" +"minForEach" +"minIf" +"minMap" +"minMappedArrays" +"minMappedArraysArgMax" +"minMappedArraysArgMin" +"minMappedArraysArray" +"minMappedArraysDistinct" +"minMappedArraysForEach" +"minMappedArraysIf" +"minMappedArraysMap" +"minMappedArraysMerge" +"minMappedArraysNull" +"minMappedArraysOrDefault" +"minMappedArraysOrNull" +"minMappedArraysResample" +"minMappedArraysSimpleState" +"minMappedArraysState" +"minMerge" +"minNull" +"minOrDefault" +"minOrNull" +"minResample" +"minSampleSizeContinous" +"minSampleSizeContinuous" +"minSampleSizeConversion" +"minSimpleState" +"minState" +"minus" +"mismatches" +"mod" +"modulo" +"moduloLegacy" +"moduloOrZero" +"monthName" +"mortonDecode" +"mortonEncode" +"multiFuzzyMatchAllIndices" +"multiFuzzyMatchAny" +"multiFuzzyMatchAnyIndex" +"multiIf" +"multiMatchAllIndices" +"multiMatchAny" +"multiMatchAnyIndex" +"multiSearchAllPositions" +"multiSearchAllPositionsCaseInsensitive" +"multiSearchAllPositionsCaseInsensitiveUTF8" +"multiSearchAllPositionsUTF8" +"multiSearchAny" +"multiSearchAnyCaseInsensitive" +"multiSearchAnyCaseInsensitiveUTF8" +"multiSearchAnyUTF8" +"multiSearchFirstIndex" +"multiSearchFirstIndexCaseInsensitive" +"multiSearchFirstIndexCaseInsensitiveUTF8" +"multiSearchFirstIndexUTF8" +"multiSearchFirstPosition" +"multiSearchFirstPositionCaseInsensitive" +"multiSearchFirstPositionCaseInsensitiveUTF8" +"multiSearchFirstPositionUTF8" +"multiply" +"multiplyDecimal" +"murmurHash2_32" +"murmurHash2_64" +"murmurHash3_128" +"murmurHash3_32" +"murmurHash3_64" +"negate" +"neighbor" +"nested" +"netloc" +"ngramDistance" +"ngramDistanceCaseInsensitive" +"ngramDistanceCaseInsensitiveUTF8" +"ngramDistanceUTF8" +"ngramMinHash" +"ngramMinHashArg" +"ngramMinHashArgCaseInsensitive" +"ngramMinHashArgCaseInsensitiveUTF8" +"ngramMinHashArgUTF8" +"ngramMinHashCaseInsensitive" +"ngramMinHashCaseInsensitiveUTF8" +"ngramMinHashUTF8" +"ngramSearch" +"ngramSearchCaseInsensitive" +"ngramSearchCaseInsensitiveUTF8" +"ngramSearchUTF8" +"ngramSimHash" +"ngramSimHashCaseInsensitive" +"ngramSimHashCaseInsensitiveUTF8" +"ngramSimHashUTF8" +"ngrams" +"nonNegativeDerivative" +"nonNegativeDerivativeArgMax" +"nonNegativeDerivativeArgMin" +"nonNegativeDerivativeArray" +"nonNegativeDerivativeDistinct" +"nonNegativeDerivativeForEach" +"nonNegativeDerivativeIf" +"nonNegativeDerivativeMap" +"nonNegativeDerivativeMerge" +"nonNegativeDerivativeNull" +"nonNegativeDerivativeOrDefault" +"nonNegativeDerivativeOrNull" +"nonNegativeDerivativeResample" +"nonNegativeDerivativeSimpleState" +"nonNegativeDerivativeState" +"normL1" +"normL2" +"normL2Squared" +"normLinf" +"normLp" +"normalizeL1" +"normalizeL2" +"normalizeLinf" +"normalizeLp" +"normalizeQuery" +"normalizeQueryKeepNames" +"normalizeUTF8NFC" +"normalizeUTF8NFD" +"normalizeUTF8NFKC" +"normalizeUTF8NFKD" +"normalizedQueryHash" +"normalizedQueryHashKeepNames" +"not" +"notEmpty" +"notEquals" +"notILike" +"notIn" +"notInIgnoreSet" +"notLike" +"notNullIn" +"notNullInIgnoreSet" +"nothing" +"nothingArgMax" +"nothingArgMin" +"nothingArray" +"nothingDistinct" +"nothingForEach" +"nothingIf" +"nothingMap" +"nothingMerge" +"nothingNull" +"nothingNull" +"nothingNullArgMax" +"nothingNullArgMin" +"nothingNullArray" +"nothingNullDistinct" +"nothingNullForEach" +"nothingNullIf" +"nothingNullMap" +"nothingNullMerge" +"nothingNullNull" +"nothingNullOrDefault" +"nothingNullOrNull" +"nothingNullResample" +"nothingNullSimpleState" +"nothingNullState" +"nothingOrDefault" +"nothingOrNull" +"nothingResample" +"nothingSimpleState" +"nothingState" +"nothingUInt64" +"nothingUInt64ArgMax" +"nothingUInt64ArgMin" +"nothingUInt64Array" +"nothingUInt64Distinct" +"nothingUInt64ForEach" +"nothingUInt64If" +"nothingUInt64Map" +"nothingUInt64Merge" +"nothingUInt64Null" +"nothingUInt64OrDefault" +"nothingUInt64OrNull" +"nothingUInt64Resample" +"nothingUInt64SimpleState" +"nothingUInt64State" +"now" +"now64" +"nowInBlock" +"nth_value" +"nth_valueArgMax" +"nth_valueArgMin" +"nth_valueArray" +"nth_valueDistinct" +"nth_valueForEach" +"nth_valueIf" +"nth_valueMap" +"nth_valueMerge" +"nth_valueNull" +"nth_valueOrDefault" +"nth_valueOrNull" +"nth_valueResample" +"nth_valueSimpleState" +"nth_valueState" +"ntile" +"ntileArgMax" +"ntileArgMin" +"ntileArray" +"ntileDistinct" +"ntileForEach" +"ntileIf" +"ntileMap" +"ntileMerge" +"ntileNull" +"ntileOrDefault" +"ntileOrNull" +"ntileResample" +"ntileSimpleState" +"ntileState" +"nullIf" +"nullIn" +"nullInIgnoreSet" +"or" +"parseDateTime" +"parseDateTime32BestEffort" +"parseDateTime32BestEffortOrNull" +"parseDateTime32BestEffortOrZero" +"parseDateTime64BestEffort" +"parseDateTime64BestEffortOrNull" +"parseDateTime64BestEffortOrZero" +"parseDateTime64BestEffortUS" +"parseDateTime64BestEffortUSOrNull" +"parseDateTime64BestEffortUSOrZero" +"parseDateTimeBestEffort" +"parseDateTimeBestEffortOrNull" +"parseDateTimeBestEffortOrZero" +"parseDateTimeBestEffortUS" +"parseDateTimeBestEffortUSOrNull" +"parseDateTimeBestEffortUSOrZero" +"parseDateTimeInJodaSyntax" +"parseDateTimeInJodaSyntaxOrNull" +"parseDateTimeInJodaSyntaxOrZero" +"parseDateTimeOrNull" +"parseDateTimeOrZero" +"parseTimeDelta" +"partitionId" +"path" +"pathFull" +"pi" +"plus" +"pmod" +"pointInEllipses" +"pointInPolygon" +"polygonAreaCartesian" +"polygonAreaSpherical" +"polygonConvexHullCartesian" +"polygonPerimeterCartesian" +"polygonPerimeterSpherical" +"polygonsDistanceCartesian" +"polygonsDistanceSpherical" +"polygonsEqualsCartesian" +"polygonsIntersectionCartesian" +"polygonsIntersectionSpherical" +"polygonsSymDifferenceCartesian" +"polygonsSymDifferenceSpherical" +"polygonsUnionCartesian" +"polygonsUnionSpherical" +"polygonsWithinCartesian" +"polygonsWithinSpherical" +"port" +"portRFC" +"position" +"positionCaseInsensitive" +"positionCaseInsensitiveUTF8" +"positionUTF8" +"positiveModulo" +"positive_modulo" +"pow" +"power" +"proportionsZTest" +"protocol" +"punycodeDecode" +"punycodeEncode" "quantile" +"quantileArgMax" +"quantileArgMin" +"quantileArray" +"quantileBFloat16" +"quantileBFloat16ArgMax" +"quantileBFloat16ArgMin" +"quantileBFloat16Array" +"quantileBFloat16Distinct" +"quantileBFloat16ForEach" +"quantileBFloat16If" +"quantileBFloat16Map" +"quantileBFloat16Merge" +"quantileBFloat16Null" +"quantileBFloat16OrDefault" +"quantileBFloat16OrNull" +"quantileBFloat16Resample" +"quantileBFloat16SimpleState" +"quantileBFloat16State" +"quantileBFloat16Weighted" +"quantileBFloat16WeightedArgMax" +"quantileBFloat16WeightedArgMin" +"quantileBFloat16WeightedArray" +"quantileBFloat16WeightedDistinct" +"quantileBFloat16WeightedForEach" +"quantileBFloat16WeightedIf" +"quantileBFloat16WeightedMap" +"quantileBFloat16WeightedMerge" +"quantileBFloat16WeightedNull" +"quantileBFloat16WeightedOrDefault" +"quantileBFloat16WeightedOrNull" +"quantileBFloat16WeightedResample" +"quantileBFloat16WeightedSimpleState" +"quantileBFloat16WeightedState" +"quantileDD" +"quantileDDArgMax" +"quantileDDArgMin" +"quantileDDArray" +"quantileDDDistinct" +"quantileDDForEach" +"quantileDDIf" +"quantileDDMap" +"quantileDDMerge" +"quantileDDNull" +"quantileDDOrDefault" +"quantileDDOrNull" +"quantileDDResample" +"quantileDDSimpleState" +"quantileDDState" +"quantileDeterministic" +"quantileDeterministicArgMax" +"quantileDeterministicArgMin" +"quantileDeterministicArray" +"quantileDeterministicDistinct" +"quantileDeterministicForEach" +"quantileDeterministicIf" +"quantileDeterministicMap" +"quantileDeterministicMerge" +"quantileDeterministicNull" +"quantileDeterministicOrDefault" +"quantileDeterministicOrNull" +"quantileDeterministicResample" +"quantileDeterministicSimpleState" +"quantileDeterministicState" +"quantileDistinct" +"quantileExact" +"quantileExactArgMax" +"quantileExactArgMin" +"quantileExactArray" +"quantileExactDistinct" +"quantileExactExclusive" +"quantileExactExclusiveArgMax" +"quantileExactExclusiveArgMin" +"quantileExactExclusiveArray" +"quantileExactExclusiveDistinct" +"quantileExactExclusiveForEach" +"quantileExactExclusiveIf" +"quantileExactExclusiveMap" +"quantileExactExclusiveMerge" +"quantileExactExclusiveNull" +"quantileExactExclusiveOrDefault" +"quantileExactExclusiveOrNull" +"quantileExactExclusiveResample" +"quantileExactExclusiveSimpleState" +"quantileExactExclusiveState" +"quantileExactForEach" +"quantileExactHigh" +"quantileExactHighArgMax" +"quantileExactHighArgMin" +"quantileExactHighArray" +"quantileExactHighDistinct" +"quantileExactHighForEach" +"quantileExactHighIf" +"quantileExactHighMap" +"quantileExactHighMerge" +"quantileExactHighNull" +"quantileExactHighOrDefault" +"quantileExactHighOrNull" +"quantileExactHighResample" +"quantileExactHighSimpleState" +"quantileExactHighState" +"quantileExactIf" +"quantileExactInclusive" +"quantileExactInclusiveArgMax" +"quantileExactInclusiveArgMin" +"quantileExactInclusiveArray" +"quantileExactInclusiveDistinct" +"quantileExactInclusiveForEach" +"quantileExactInclusiveIf" +"quantileExactInclusiveMap" +"quantileExactInclusiveMerge" +"quantileExactInclusiveNull" +"quantileExactInclusiveOrDefault" +"quantileExactInclusiveOrNull" +"quantileExactInclusiveResample" +"quantileExactInclusiveSimpleState" +"quantileExactInclusiveState" +"quantileExactLow" +"quantileExactLowArgMax" +"quantileExactLowArgMin" +"quantileExactLowArray" +"quantileExactLowDistinct" +"quantileExactLowForEach" +"quantileExactLowIf" +"quantileExactLowMap" +"quantileExactLowMerge" +"quantileExactLowNull" +"quantileExactLowOrDefault" +"quantileExactLowOrNull" +"quantileExactLowResample" +"quantileExactLowSimpleState" +"quantileExactLowState" +"quantileExactMap" +"quantileExactMerge" +"quantileExactNull" +"quantileExactOrDefault" +"quantileExactOrNull" +"quantileExactResample" +"quantileExactSimpleState" +"quantileExactState" +"quantileExactWeighted" +"quantileExactWeightedArgMax" +"quantileExactWeightedArgMin" +"quantileExactWeightedArray" +"quantileExactWeightedDistinct" +"quantileExactWeightedForEach" +"quantileExactWeightedIf" +"quantileExactWeightedMap" +"quantileExactWeightedMerge" +"quantileExactWeightedNull" +"quantileExactWeightedOrDefault" +"quantileExactWeightedOrNull" +"quantileExactWeightedResample" +"quantileExactWeightedSimpleState" +"quantileExactWeightedState" +"quantileForEach" +"quantileGK" +"quantileGKArgMax" +"quantileGKArgMin" +"quantileGKArray" +"quantileGKDistinct" +"quantileGKForEach" +"quantileGKIf" +"quantileGKMap" +"quantileGKMerge" +"quantileGKNull" +"quantileGKOrDefault" +"quantileGKOrNull" +"quantileGKResample" +"quantileGKSimpleState" +"quantileGKState" +"quantileIf" +"quantileInterpolatedWeighted" +"quantileInterpolatedWeightedArgMax" +"quantileInterpolatedWeightedArgMin" +"quantileInterpolatedWeightedArray" +"quantileInterpolatedWeightedDistinct" +"quantileInterpolatedWeightedForEach" +"quantileInterpolatedWeightedIf" +"quantileInterpolatedWeightedMap" +"quantileInterpolatedWeightedMerge" +"quantileInterpolatedWeightedNull" +"quantileInterpolatedWeightedOrDefault" +"quantileInterpolatedWeightedOrNull" +"quantileInterpolatedWeightedResample" +"quantileInterpolatedWeightedSimpleState" +"quantileInterpolatedWeightedState" +"quantileMap" +"quantileMerge" +"quantileNull" +"quantileOrDefault" +"quantileOrNull" +"quantileResample" +"quantileSimpleState" +"quantileState" +"quantileTDigest" +"quantileTDigestArgMax" +"quantileTDigestArgMin" +"quantileTDigestArray" +"quantileTDigestDistinct" +"quantileTDigestForEach" +"quantileTDigestIf" +"quantileTDigestMap" +"quantileTDigestMerge" +"quantileTDigestNull" +"quantileTDigestOrDefault" +"quantileTDigestOrNull" +"quantileTDigestResample" +"quantileTDigestSimpleState" +"quantileTDigestState" +"quantileTDigestWeighted" +"quantileTDigestWeightedArgMax" +"quantileTDigestWeightedArgMin" +"quantileTDigestWeightedArray" +"quantileTDigestWeightedDistinct" +"quantileTDigestWeightedForEach" +"quantileTDigestWeightedIf" +"quantileTDigestWeightedMap" +"quantileTDigestWeightedMerge" +"quantileTDigestWeightedNull" +"quantileTDigestWeightedOrDefault" +"quantileTDigestWeightedOrNull" +"quantileTDigestWeightedResample" +"quantileTDigestWeightedSimpleState" +"quantileTDigestWeightedState" +"quantileTiming" +"quantileTimingArgMax" +"quantileTimingArgMin" +"quantileTimingArray" +"quantileTimingDistinct" +"quantileTimingForEach" +"quantileTimingIf" +"quantileTimingMap" +"quantileTimingMerge" +"quantileTimingNull" +"quantileTimingOrDefault" +"quantileTimingOrNull" +"quantileTimingResample" +"quantileTimingSimpleState" +"quantileTimingState" +"quantileTimingWeighted" +"quantileTimingWeightedArgMax" +"quantileTimingWeightedArgMin" +"quantileTimingWeightedArray" +"quantileTimingWeightedDistinct" +"quantileTimingWeightedForEach" +"quantileTimingWeightedIf" +"quantileTimingWeightedMap" +"quantileTimingWeightedMerge" +"quantileTimingWeightedNull" +"quantileTimingWeightedOrDefault" +"quantileTimingWeightedOrNull" +"quantileTimingWeightedResample" +"quantileTimingWeightedSimpleState" +"quantileTimingWeightedState" +"quantiles" +"quantilesArgMax" +"quantilesArgMin" +"quantilesArray" +"quantilesBFloat16" +"quantilesBFloat16ArgMax" +"quantilesBFloat16ArgMin" +"quantilesBFloat16Array" +"quantilesBFloat16Distinct" +"quantilesBFloat16ForEach" +"quantilesBFloat16If" +"quantilesBFloat16Map" +"quantilesBFloat16Merge" +"quantilesBFloat16Null" +"quantilesBFloat16OrDefault" +"quantilesBFloat16OrNull" +"quantilesBFloat16Resample" +"quantilesBFloat16SimpleState" +"quantilesBFloat16State" +"quantilesBFloat16Weighted" +"quantilesBFloat16WeightedArgMax" +"quantilesBFloat16WeightedArgMin" +"quantilesBFloat16WeightedArray" +"quantilesBFloat16WeightedDistinct" +"quantilesBFloat16WeightedForEach" +"quantilesBFloat16WeightedIf" +"quantilesBFloat16WeightedMap" +"quantilesBFloat16WeightedMerge" +"quantilesBFloat16WeightedNull" +"quantilesBFloat16WeightedOrDefault" +"quantilesBFloat16WeightedOrNull" +"quantilesBFloat16WeightedResample" +"quantilesBFloat16WeightedSimpleState" +"quantilesBFloat16WeightedState" +"quantilesDD" +"quantilesDDArgMax" +"quantilesDDArgMin" +"quantilesDDArray" +"quantilesDDDistinct" +"quantilesDDForEach" +"quantilesDDIf" +"quantilesDDMap" +"quantilesDDMerge" +"quantilesDDNull" +"quantilesDDOrDefault" +"quantilesDDOrNull" +"quantilesDDResample" +"quantilesDDSimpleState" +"quantilesDDState" +"quantilesDeterministic" +"quantilesDeterministicArgMax" +"quantilesDeterministicArgMin" +"quantilesDeterministicArray" +"quantilesDeterministicDistinct" +"quantilesDeterministicForEach" +"quantilesDeterministicIf" +"quantilesDeterministicMap" +"quantilesDeterministicMerge" +"quantilesDeterministicNull" +"quantilesDeterministicOrDefault" +"quantilesDeterministicOrNull" +"quantilesDeterministicResample" +"quantilesDeterministicSimpleState" +"quantilesDeterministicState" +"quantilesDistinct" +"quantilesExact" +"quantilesExactArgMax" +"quantilesExactArgMin" +"quantilesExactArray" +"quantilesExactDistinct" +"quantilesExactExclusive" +"quantilesExactExclusiveArgMax" +"quantilesExactExclusiveArgMin" +"quantilesExactExclusiveArray" +"quantilesExactExclusiveDistinct" +"quantilesExactExclusiveForEach" +"quantilesExactExclusiveIf" +"quantilesExactExclusiveMap" +"quantilesExactExclusiveMerge" +"quantilesExactExclusiveNull" +"quantilesExactExclusiveOrDefault" +"quantilesExactExclusiveOrNull" +"quantilesExactExclusiveResample" +"quantilesExactExclusiveSimpleState" +"quantilesExactExclusiveState" +"quantilesExactForEach" +"quantilesExactHigh" +"quantilesExactHighArgMax" +"quantilesExactHighArgMin" +"quantilesExactHighArray" +"quantilesExactHighDistinct" +"quantilesExactHighForEach" +"quantilesExactHighIf" +"quantilesExactHighMap" +"quantilesExactHighMerge" +"quantilesExactHighNull" +"quantilesExactHighOrDefault" +"quantilesExactHighOrNull" +"quantilesExactHighResample" +"quantilesExactHighSimpleState" +"quantilesExactHighState" +"quantilesExactIf" +"quantilesExactInclusive" +"quantilesExactInclusiveArgMax" +"quantilesExactInclusiveArgMin" +"quantilesExactInclusiveArray" +"quantilesExactInclusiveDistinct" +"quantilesExactInclusiveForEach" +"quantilesExactInclusiveIf" +"quantilesExactInclusiveMap" +"quantilesExactInclusiveMerge" +"quantilesExactInclusiveNull" +"quantilesExactInclusiveOrDefault" +"quantilesExactInclusiveOrNull" +"quantilesExactInclusiveResample" +"quantilesExactInclusiveSimpleState" +"quantilesExactInclusiveState" +"quantilesExactLow" +"quantilesExactLowArgMax" +"quantilesExactLowArgMin" +"quantilesExactLowArray" +"quantilesExactLowDistinct" +"quantilesExactLowForEach" +"quantilesExactLowIf" +"quantilesExactLowMap" +"quantilesExactLowMerge" +"quantilesExactLowNull" +"quantilesExactLowOrDefault" +"quantilesExactLowOrNull" +"quantilesExactLowResample" +"quantilesExactLowSimpleState" +"quantilesExactLowState" +"quantilesExactMap" +"quantilesExactMerge" +"quantilesExactNull" +"quantilesExactOrDefault" +"quantilesExactOrNull" +"quantilesExactResample" +"quantilesExactSimpleState" +"quantilesExactState" +"quantilesExactWeighted" +"quantilesExactWeightedArgMax" +"quantilesExactWeightedArgMin" +"quantilesExactWeightedArray" +"quantilesExactWeightedDistinct" +"quantilesExactWeightedForEach" +"quantilesExactWeightedIf" +"quantilesExactWeightedMap" +"quantilesExactWeightedMerge" +"quantilesExactWeightedNull" +"quantilesExactWeightedOrDefault" +"quantilesExactWeightedOrNull" +"quantilesExactWeightedResample" +"quantilesExactWeightedSimpleState" +"quantilesExactWeightedState" +"quantilesForEach" +"quantilesGK" +"quantilesGKArgMax" +"quantilesGKArgMin" +"quantilesGKArray" +"quantilesGKDistinct" +"quantilesGKForEach" +"quantilesGKIf" +"quantilesGKMap" +"quantilesGKMerge" +"quantilesGKNull" +"quantilesGKOrDefault" +"quantilesGKOrNull" +"quantilesGKResample" +"quantilesGKSimpleState" +"quantilesGKState" +"quantilesIf" +"quantilesInterpolatedWeighted" +"quantilesInterpolatedWeightedArgMax" +"quantilesInterpolatedWeightedArgMin" +"quantilesInterpolatedWeightedArray" +"quantilesInterpolatedWeightedDistinct" +"quantilesInterpolatedWeightedForEach" +"quantilesInterpolatedWeightedIf" +"quantilesInterpolatedWeightedMap" +"quantilesInterpolatedWeightedMerge" +"quantilesInterpolatedWeightedNull" +"quantilesInterpolatedWeightedOrDefault" +"quantilesInterpolatedWeightedOrNull" +"quantilesInterpolatedWeightedResample" +"quantilesInterpolatedWeightedSimpleState" +"quantilesInterpolatedWeightedState" +"quantilesMap" +"quantilesMerge" +"quantilesNull" +"quantilesOrDefault" +"quantilesOrNull" +"quantilesResample" +"quantilesSimpleState" +"quantilesState" +"quantilesTDigest" +"quantilesTDigestArgMax" +"quantilesTDigestArgMin" +"quantilesTDigestArray" +"quantilesTDigestDistinct" +"quantilesTDigestForEach" +"quantilesTDigestIf" +"quantilesTDigestMap" +"quantilesTDigestMerge" +"quantilesTDigestNull" +"quantilesTDigestOrDefault" +"quantilesTDigestOrNull" +"quantilesTDigestResample" +"quantilesTDigestSimpleState" +"quantilesTDigestState" +"quantilesTDigestWeighted" +"quantilesTDigestWeightedArgMax" +"quantilesTDigestWeightedArgMin" +"quantilesTDigestWeightedArray" +"quantilesTDigestWeightedDistinct" +"quantilesTDigestWeightedForEach" +"quantilesTDigestWeightedIf" +"quantilesTDigestWeightedMap" +"quantilesTDigestWeightedMerge" +"quantilesTDigestWeightedNull" +"quantilesTDigestWeightedOrDefault" +"quantilesTDigestWeightedOrNull" +"quantilesTDigestWeightedResample" +"quantilesTDigestWeightedSimpleState" +"quantilesTDigestWeightedState" +"quantilesTiming" +"quantilesTimingArgMax" +"quantilesTimingArgMin" +"quantilesTimingArray" +"quantilesTimingDistinct" +"quantilesTimingForEach" +"quantilesTimingIf" +"quantilesTimingMap" +"quantilesTimingMerge" +"quantilesTimingNull" +"quantilesTimingOrDefault" +"quantilesTimingOrNull" +"quantilesTimingResample" +"quantilesTimingSimpleState" +"quantilesTimingState" +"quantilesTimingWeighted" +"quantilesTimingWeightedArgMax" +"quantilesTimingWeightedArgMin" +"quantilesTimingWeightedArray" +"quantilesTimingWeightedDistinct" +"quantilesTimingWeightedForEach" +"quantilesTimingWeightedIf" +"quantilesTimingWeightedMap" +"quantilesTimingWeightedMerge" +"quantilesTimingWeightedNull" +"quantilesTimingWeightedOrDefault" +"quantilesTimingWeightedOrNull" +"quantilesTimingWeightedResample" +"quantilesTimingWeightedSimpleState" +"quantilesTimingWeightedState" +"queryID" +"queryString" +"queryStringAndFragment" +"query_id" +"radians" +"rand" +"rand32" +"rand64" +"randBernoulli" +"randBinomial" +"randCanonical" +"randChiSquared" +"randConstant" +"randExponential" +"randFisherF" +"randLogNormal" +"randNegativeBinomial" +"randNormal" +"randPoisson" +"randStudentT" +"randUniform" +"randomFixedString" +"randomPrintableASCII" +"randomString" +"randomStringUTF8" +"range" +"rank" +"rankArgMax" +"rankArgMin" +"rankArray" +"rankCorr" +"rankCorrArgMax" +"rankCorrArgMin" +"rankCorrArray" +"rankCorrDistinct" +"rankCorrForEach" +"rankCorrIf" +"rankCorrMap" +"rankCorrMerge" +"rankCorrNull" +"rankCorrOrDefault" +"rankCorrOrNull" +"rankCorrResample" +"rankCorrSimpleState" +"rankCorrState" +"rankDistinct" +"rankForEach" +"rankIf" +"rankMap" +"rankMerge" +"rankNull" +"rankOrDefault" +"rankOrNull" +"rankResample" +"rankSimpleState" +"rankState" +"readWKTMultiPolygon" +"readWKTPoint" +"readWKTPolygon" +"readWKTRing" +"regexpExtract" +"regexpQuoteMeta" +"regionHierarchy" +"regionIn" +"regionToArea" +"regionToCity" +"regionToContinent" +"regionToCountry" +"regionToDistrict" +"regionToName" +"regionToPopulation" +"regionToTopContinent" +"reinterpret" +"reinterpretAsDate" +"reinterpretAsDateTime" +"reinterpretAsFixedString" +"reinterpretAsFloat32" +"reinterpretAsFloat64" +"reinterpretAsInt128" +"reinterpretAsInt16" +"reinterpretAsInt256" +"reinterpretAsInt32" +"reinterpretAsInt64" +"reinterpretAsInt8" +"reinterpretAsString" +"reinterpretAsUInt128" +"reinterpretAsUInt16" +"reinterpretAsUInt256" +"reinterpretAsUInt32" +"reinterpretAsUInt64" +"reinterpretAsUInt8" +"reinterpretAsUUID" +"repeat" +"replace" +"replaceAll" +"replaceOne" +"replaceRegexpAll" +"replaceRegexpOne" +"replicate" +"retention" +"retentionArgMax" +"retentionArgMin" +"retentionArray" +"retentionDistinct" +"retentionForEach" +"retentionIf" +"retentionMap" +"retentionMerge" +"retentionNull" +"retentionOrDefault" +"retentionOrNull" +"retentionResample" +"retentionSimpleState" +"retentionState" +"reverse" +"reverseUTF8" +"revision" +"right" +"rightPad" +"rightPadUTF8" +"rightUTF8" +"round" +"roundAge" +"roundBankers" +"roundDown" +"roundDuration" +"roundToExp2" +"rowNumberInAllBlocks" +"rowNumberInBlock" +"row_number" +"row_numberArgMax" +"row_numberArgMin" +"row_numberArray" +"row_numberDistinct" +"row_numberForEach" +"row_numberIf" +"row_numberMap" +"row_numberMerge" +"row_numberNull" +"row_numberOrDefault" +"row_numberOrNull" +"row_numberResample" +"row_numberSimpleState" +"row_numberState" +"rpad" +"rtrim" +"runningAccumulate" +"runningConcurrency" +"runningDifference" +"runningDifferenceStartingWithFirstValue" +"s2CapContains" +"s2CapUnion" +"s2CellsIntersect" +"s2GetNeighbors" +"s2RectAdd" +"s2RectContains" +"s2RectIntersection" +"s2RectUnion" +"s2ToGeo" +"scalarProduct" +"sequenceCount" +"sequenceCountArgMax" +"sequenceCountArgMin" +"sequenceCountArray" +"sequenceCountDistinct" +"sequenceCountForEach" +"sequenceCountIf" +"sequenceCountMap" +"sequenceCountMerge" +"sequenceCountNull" +"sequenceCountOrDefault" +"sequenceCountOrNull" +"sequenceCountResample" +"sequenceCountSimpleState" +"sequenceCountState" +"sequenceMatch" +"sequenceMatchArgMax" +"sequenceMatchArgMin" +"sequenceMatchArray" +"sequenceMatchDistinct" +"sequenceMatchForEach" +"sequenceMatchIf" +"sequenceMatchMap" +"sequenceMatchMerge" +"sequenceMatchNull" +"sequenceMatchOrDefault" +"sequenceMatchOrNull" +"sequenceMatchResample" +"sequenceMatchSimpleState" +"sequenceMatchState" +"sequenceNextNode" +"sequenceNextNodeArgMax" +"sequenceNextNodeArgMin" +"sequenceNextNodeArray" +"sequenceNextNodeDistinct" +"sequenceNextNodeForEach" +"sequenceNextNodeIf" +"sequenceNextNodeMap" +"sequenceNextNodeMerge" +"sequenceNextNodeNull" +"sequenceNextNodeOrDefault" +"sequenceNextNodeOrNull" +"sequenceNextNodeResample" +"sequenceNextNodeSimpleState" +"sequenceNextNodeState" +"seriesDecomposeSTL" +"seriesOutliersDetectTukey" +"seriesPeriodDetectFFT" +"serverTimeZone" +"serverTimezone" +"serverUUID" +"shardCount" +"shardNum" +"showCertificate" +"sigmoid" +"sign" +"simpleJSONExtractBool" +"simpleJSONExtractFloat" +"simpleJSONExtractInt" +"simpleJSONExtractRaw" +"simpleJSONExtractString" +"simpleJSONExtractUInt" +"simpleJSONHas" +"simpleLinearRegression" +"simpleLinearRegressionArgMax" +"simpleLinearRegressionArgMin" +"simpleLinearRegressionArray" +"simpleLinearRegressionDistinct" +"simpleLinearRegressionForEach" +"simpleLinearRegressionIf" +"simpleLinearRegressionMap" +"simpleLinearRegressionMerge" +"simpleLinearRegressionNull" +"simpleLinearRegressionOrDefault" +"simpleLinearRegressionOrNull" +"simpleLinearRegressionResample" +"simpleLinearRegressionSimpleState" +"simpleLinearRegressionState" +"sin" +"singleValueOrNull" +"singleValueOrNullArgMax" +"singleValueOrNullArgMin" +"singleValueOrNullArray" +"singleValueOrNullDistinct" +"singleValueOrNullForEach" +"singleValueOrNullIf" +"singleValueOrNullMap" +"singleValueOrNullMerge" +"singleValueOrNullNull" +"singleValueOrNullOrDefault" +"singleValueOrNullOrNull" +"singleValueOrNullResample" +"singleValueOrNullSimpleState" +"singleValueOrNullState" +"sinh" +"sipHash128" +"sipHash128Keyed" +"sipHash128Reference" +"sipHash128ReferenceKeyed" +"sipHash64" +"sipHash64Keyed" +"skewPop" +"skewPopArgMax" +"skewPopArgMin" +"skewPopArray" +"skewPopDistinct" +"skewPopForEach" +"skewPopIf" +"skewPopMap" +"skewPopMerge" +"skewPopNull" +"skewPopOrDefault" +"skewPopOrNull" +"skewPopResample" +"skewPopSimpleState" +"skewPopState" +"skewSamp" +"skewSampArgMax" +"skewSampArgMin" +"skewSampArray" +"skewSampDistinct" +"skewSampForEach" +"skewSampIf" +"skewSampMap" +"skewSampMerge" +"skewSampNull" +"skewSampOrDefault" +"skewSampOrNull" +"skewSampResample" +"skewSampSimpleState" +"skewSampState" +"sleep" +"sleepEachRow" +"snowflakeToDateTime" +"snowflakeToDateTime64" +"soundex" +"space" +"sparkBar" +"sparkBarArgMax" +"sparkBarArgMin" +"sparkBarArray" +"sparkBarDistinct" +"sparkBarForEach" +"sparkBarIf" +"sparkBarMap" +"sparkBarMerge" +"sparkBarNull" +"sparkBarOrDefault" +"sparkBarOrNull" +"sparkBarResample" +"sparkBarSimpleState" +"sparkBarState" +"sparkbar" +"sparkbarArgMax" +"sparkbarArgMin" +"sparkbarArray" +"sparkbarDistinct" +"sparkbarForEach" +"sparkbarIf" +"sparkbarMap" +"sparkbarMerge" +"sparkbarNull" +"sparkbarOrDefault" +"sparkbarOrNull" +"sparkbarResample" +"sparkbarSimpleState" +"sparkbarState" +"splitByAlpha" +"splitByChar" +"splitByNonAlpha" +"splitByRegexp" +"splitByString" +"splitByWhitespace" +"sqid" +"sqidDecode" +"sqidEncode" +"sqrt" +"startsWith" +"startsWithUTF8" +"stddevPop" +"stddevPopArgMax" +"stddevPopArgMin" +"stddevPopArray" +"stddevPopDistinct" +"stddevPopForEach" +"stddevPopIf" +"stddevPopMap" +"stddevPopMerge" +"stddevPopNull" +"stddevPopOrDefault" +"stddevPopOrNull" +"stddevPopResample" +"stddevPopSimpleState" +"stddevPopStable" +"stddevPopStableArgMax" +"stddevPopStableArgMin" +"stddevPopStableArray" +"stddevPopStableDistinct" +"stddevPopStableForEach" +"stddevPopStableIf" +"stddevPopStableMap" +"stddevPopStableMerge" +"stddevPopStableNull" +"stddevPopStableOrDefault" +"stddevPopStableOrNull" +"stddevPopStableResample" +"stddevPopStableSimpleState" +"stddevPopStableState" +"stddevPopState" +"stddevSamp" +"stddevSampArgMax" +"stddevSampArgMin" +"stddevSampArray" +"stddevSampDistinct" +"stddevSampForEach" +"stddevSampIf" +"stddevSampMap" +"stddevSampMerge" +"stddevSampNull" +"stddevSampOrDefault" +"stddevSampOrNull" +"stddevSampResample" +"stddevSampSimpleState" +"stddevSampStable" +"stddevSampStableArgMax" +"stddevSampStableArgMin" +"stddevSampStableArray" +"stddevSampStableDistinct" +"stddevSampStableForEach" +"stddevSampStableIf" +"stddevSampStableMap" +"stddevSampStableMerge" +"stddevSampStableNull" +"stddevSampStableOrDefault" +"stddevSampStableOrNull" +"stddevSampStableResample" +"stddevSampStableSimpleState" +"stddevSampStableState" +"stddevSampState" +"stem" +"stochasticLinearRegression" +"stochasticLinearRegressionArgMax" +"stochasticLinearRegressionArgMin" +"stochasticLinearRegressionArray" +"stochasticLinearRegressionDistinct" +"stochasticLinearRegressionForEach" +"stochasticLinearRegressionIf" +"stochasticLinearRegressionMap" +"stochasticLinearRegressionMerge" +"stochasticLinearRegressionNull" +"stochasticLinearRegressionOrDefault" +"stochasticLinearRegressionOrNull" +"stochasticLinearRegressionResample" +"stochasticLinearRegressionSimpleState" +"stochasticLinearRegressionState" +"stochasticLogisticRegression" +"stochasticLogisticRegressionArgMax" +"stochasticLogisticRegressionArgMin" +"stochasticLogisticRegressionArray" +"stochasticLogisticRegressionDistinct" +"stochasticLogisticRegressionForEach" +"stochasticLogisticRegressionIf" +"stochasticLogisticRegressionMap" +"stochasticLogisticRegressionMerge" +"stochasticLogisticRegressionNull" +"stochasticLogisticRegressionOrDefault" +"stochasticLogisticRegressionOrNull" +"stochasticLogisticRegressionResample" +"stochasticLogisticRegressionSimpleState" +"stochasticLogisticRegressionState" +"str_to_date" +"str_to_map" +"stringJaccardIndex" +"stringJaccardIndexUTF8" +"stringToH3" +"structureToCapnProtoSchema" +"structureToProtobufSchema" +"studentTTest" +"studentTTestArgMax" +"studentTTestArgMin" +"studentTTestArray" +"studentTTestDistinct" +"studentTTestForEach" +"studentTTestIf" +"studentTTestMap" +"studentTTestMerge" +"studentTTestNull" +"studentTTestOrDefault" +"studentTTestOrNull" +"studentTTestResample" +"studentTTestSimpleState" +"studentTTestState" +"subBitmap" +"subDate" +"substr" +"substring" +"substringIndex" +"substringIndexUTF8" +"substringUTF8" +"subtractDays" +"subtractHours" +"subtractInterval" +"subtractMicroseconds" +"subtractMilliseconds" +"subtractMinutes" +"subtractMonths" +"subtractNanoseconds" +"subtractQuarters" +"subtractSeconds" +"subtractTupleOfIntervals" +"subtractWeeks" +"subtractYears" +"sum" +"sumArgMax" +"sumArgMin" +"sumArray" +"sumCount" +"sumCountArgMax" +"sumCountArgMin" +"sumCountArray" +"sumCountDistinct" +"sumCountForEach" +"sumCountIf" +"sumCountMap" +"sumCountMerge" +"sumCountNull" +"sumCountOrDefault" +"sumCountOrNull" +"sumCountResample" +"sumCountSimpleState" +"sumCountState" +"sumDistinct" +"sumForEach" +"sumIf" +"sumKahan" +"sumKahanArgMax" +"sumKahanArgMin" +"sumKahanArray" +"sumKahanDistinct" +"sumKahanForEach" +"sumKahanIf" +"sumKahanMap" +"sumKahanMerge" +"sumKahanNull" +"sumKahanOrDefault" +"sumKahanOrNull" +"sumKahanResample" +"sumKahanSimpleState" +"sumKahanState" +"sumMap" +"sumMapFiltered" +"sumMapFilteredArgMax" +"sumMapFilteredArgMin" +"sumMapFilteredArray" +"sumMapFilteredDistinct" +"sumMapFilteredForEach" +"sumMapFilteredIf" +"sumMapFilteredMap" +"sumMapFilteredMerge" +"sumMapFilteredNull" +"sumMapFilteredOrDefault" +"sumMapFilteredOrNull" +"sumMapFilteredResample" +"sumMapFilteredSimpleState" +"sumMapFilteredState" +"sumMapFilteredWithOverflow" +"sumMapFilteredWithOverflowArgMax" +"sumMapFilteredWithOverflowArgMin" +"sumMapFilteredWithOverflowArray" +"sumMapFilteredWithOverflowDistinct" +"sumMapFilteredWithOverflowForEach" +"sumMapFilteredWithOverflowIf" +"sumMapFilteredWithOverflowMap" +"sumMapFilteredWithOverflowMerge" +"sumMapFilteredWithOverflowNull" +"sumMapFilteredWithOverflowOrDefault" +"sumMapFilteredWithOverflowOrNull" +"sumMapFilteredWithOverflowResample" +"sumMapFilteredWithOverflowSimpleState" +"sumMapFilteredWithOverflowState" +"sumMapWithOverflow" +"sumMapWithOverflowArgMax" +"sumMapWithOverflowArgMin" +"sumMapWithOverflowArray" +"sumMapWithOverflowDistinct" +"sumMapWithOverflowForEach" +"sumMapWithOverflowIf" +"sumMapWithOverflowMap" +"sumMapWithOverflowMerge" +"sumMapWithOverflowNull" +"sumMapWithOverflowOrDefault" +"sumMapWithOverflowOrNull" +"sumMapWithOverflowResample" +"sumMapWithOverflowSimpleState" +"sumMapWithOverflowState" +"sumMappedArrays" +"sumMappedArraysArgMax" +"sumMappedArraysArgMin" +"sumMappedArraysArray" +"sumMappedArraysDistinct" +"sumMappedArraysForEach" +"sumMappedArraysIf" +"sumMappedArraysMap" +"sumMappedArraysMerge" +"sumMappedArraysNull" +"sumMappedArraysOrDefault" +"sumMappedArraysOrNull" +"sumMappedArraysResample" +"sumMappedArraysSimpleState" +"sumMappedArraysState" +"sumMerge" +"sumNull" +"sumOrDefault" +"sumOrNull" +"sumResample" +"sumSimpleState" +"sumState" +"sumWithOverflow" +"sumWithOverflowArgMax" +"sumWithOverflowArgMin" +"sumWithOverflowArray" +"sumWithOverflowDistinct" +"sumWithOverflowForEach" +"sumWithOverflowIf" +"sumWithOverflowMap" +"sumWithOverflowMerge" +"sumWithOverflowNull" +"sumWithOverflowOrDefault" +"sumWithOverflowOrNull" +"sumWithOverflowResample" +"sumWithOverflowSimpleState" +"sumWithOverflowState" +"svg" +"synonyms" +"tan" +"tanh" +"tcpPort" +"tgamma" +"theilsU" +"theilsUArgMax" +"theilsUArgMin" +"theilsUArray" +"theilsUDistinct" +"theilsUForEach" +"theilsUIf" +"theilsUMap" +"theilsUMerge" +"theilsUNull" +"theilsUOrDefault" +"theilsUOrNull" +"theilsUResample" +"theilsUSimpleState" +"theilsUState" +"throwIf" +"tid" +"timeDiff" +"timeSlot" +"timeSlots" +"timeZone" +"timeZoneOf" +"timeZoneOffset" +"timestamp" +"timestampDiff" +"timestamp_diff" +"timezone" +"timezoneOf" +"timezoneOffset" +"toBool" +"toColumnTypeName" +"toDate" +"toDate32" +"toDate32OrDefault" +"toDate32OrNull" +"toDate32OrZero" +"toDateOrDefault" +"toDateOrNull" +"toDateOrZero" +"toDateTime" +"toDateTime32" +"toDateTime64" +"toDateTime64OrDefault" +"toDateTime64OrNull" +"toDateTime64OrZero" +"toDateTimeOrDefault" +"toDateTimeOrNull" +"toDateTimeOrZero" +"toDayOfMonth" +"toDayOfWeek" +"toDayOfYear" +"toDaysSinceYearZero" +"toDecimal128" +"toDecimal128OrDefault" +"toDecimal128OrNull" +"toDecimal128OrZero" +"toDecimal256" +"toDecimal256OrDefault" +"toDecimal256OrNull" +"toDecimal256OrZero" +"toDecimal32" +"toDecimal32OrDefault" +"toDecimal32OrNull" +"toDecimal32OrZero" +"toDecimal64" +"toDecimal64OrDefault" +"toDecimal64OrNull" +"toDecimal64OrZero" +"toDecimalString" +"toFixedString" +"toFloat32" +"toFloat32OrDefault" +"toFloat32OrNull" +"toFloat32OrZero" +"toFloat64" +"toFloat64OrDefault" +"toFloat64OrNull" +"toFloat64OrZero" +"toHour" +"toIPv4" +"toIPv4OrDefault" +"toIPv4OrNull" +"toIPv4OrZero" +"toIPv6" +"toIPv6OrDefault" +"toIPv6OrNull" +"toIPv6OrZero" +"toISOWeek" +"toISOYear" +"toInt128" +"toInt128OrDefault" +"toInt128OrNull" +"toInt128OrZero" +"toInt16" +"toInt16OrDefault" +"toInt16OrNull" +"toInt16OrZero" +"toInt256" +"toInt256OrDefault" +"toInt256OrNull" +"toInt256OrZero" +"toInt32" +"toInt32OrDefault" +"toInt32OrNull" +"toInt32OrZero" +"toInt64" +"toInt64OrDefault" +"toInt64OrNull" +"toInt64OrZero" +"toInt8" +"toInt8OrDefault" +"toInt8OrNull" +"toInt8OrZero" +"toIntervalDay" +"toIntervalHour" +"toIntervalMicrosecond" +"toIntervalMillisecond" +"toIntervalMinute" +"toIntervalMonth" +"toIntervalNanosecond" +"toIntervalQuarter" +"toIntervalSecond" +"toIntervalWeek" +"toIntervalYear" +"toJSONString" +"toLastDayOfMonth" +"toLastDayOfWeek" +"toLowCardinality" +"toMillisecond" +"toMinute" +"toModifiedJulianDay" +"toModifiedJulianDayOrNull" +"toMonday" +"toMonth" +"toNullable" +"toQuarter" +"toRelativeDayNum" +"toRelativeHourNum" +"toRelativeMinuteNum" +"toRelativeMonthNum" +"toRelativeQuarterNum" +"toRelativeSecondNum" +"toRelativeWeekNum" +"toRelativeYearNum" +"toSecond" +"toStartOfDay" +"toStartOfFifteenMinutes" +"toStartOfFiveMinute" +"toStartOfFiveMinutes" +"toStartOfHour" +"toStartOfISOYear" +"toStartOfInterval" +"toStartOfMicrosecond" +"toStartOfMillisecond" +"toStartOfMinute" +"toStartOfMonth" +"toStartOfNanosecond" +"toStartOfQuarter" +"toStartOfSecond" +"toStartOfTenMinutes" +"toStartOfWeek" +"toStartOfYear" +"toString" +"toStringCutToZero" +"toTime" +"toTimeZone" +"toTimezone" +"toTypeName" +"toUInt128" +"toUInt128OrDefault" +"toUInt128OrNull" +"toUInt128OrZero" +"toUInt16" +"toUInt16OrDefault" +"toUInt16OrNull" +"toUInt16OrZero" +"toUInt256" +"toUInt256OrDefault" +"toUInt256OrNull" +"toUInt256OrZero" +"toUInt32" +"toUInt32OrDefault" +"toUInt32OrNull" +"toUInt32OrZero" +"toUInt64" +"toUInt64OrDefault" +"toUInt64OrNull" +"toUInt64OrZero" +"toUInt8" +"toUInt8OrDefault" +"toUInt8OrNull" +"toUInt8OrZero" +"toUTCTimestamp" +"toUUID" +"toUUIDOrDefault" +"toUUIDOrNull" +"toUUIDOrZero" +"toUnixTimestamp" +"toUnixTimestamp64Micro" +"toUnixTimestamp64Milli" +"toUnixTimestamp64Nano" +"toValidUTF8" +"toWeek" +"toYYYYMM" +"toYYYYMMDD" +"toYYYYMMDDhhmmss" +"toYear" +"toYearWeek" +"to_utc_timestamp" +"today" +"tokens" +"topK" +"topKArgMax" +"topKArgMin" +"topKArray" +"topKDistinct" +"topKForEach" +"topKIf" +"topKMap" +"topKMerge" +"topKNull" +"topKOrDefault" +"topKOrNull" +"topKResample" +"topKSimpleState" +"topKState" +"topKWeighted" +"topKWeightedArgMax" +"topKWeightedArgMin" +"topKWeightedArray" +"topKWeightedDistinct" +"topKWeightedForEach" +"topKWeightedIf" +"topKWeightedMap" +"topKWeightedMerge" +"topKWeightedNull" +"topKWeightedOrDefault" +"topKWeightedOrNull" +"topKWeightedResample" +"topKWeightedSimpleState" +"topKWeightedState" +"topLevelDomain" +"topLevelDomainRFC" +"transactionID" +"transactionLatestSnapshot" +"transactionOldestSnapshot" +"transform" +"translate" +"translateUTF8" +"trim" +"trimBoth" +"trimLeft" +"trimRight" +"trunc" +"truncate" +"tryBase58Decode" +"tryBase64Decode" +"tryDecrypt" +"tryIdnaEncode" +"tryPunycodeDecode" +"tumble" +"tumbleEnd" +"tumbleStart" +"tuple" +"tupleConcat" +"tupleDivide" +"tupleDivideByNumber" +"tupleElement" +"tupleHammingDistance" +"tupleIntDiv" +"tupleIntDivByNumber" +"tupleIntDivOrZero" +"tupleIntDivOrZeroByNumber" +"tupleMinus" +"tupleModulo" +"tupleModuloByNumber" +"tupleMultiply" +"tupleMultiplyByNumber" +"tupleNegate" +"tuplePlus" +"tupleToNameValuePairs" +"ucase" +"unbin" +"unhex" +"uniq" +"uniqArgMax" +"uniqArgMin" +"uniqArray" +"uniqCombined" +"uniqCombined64" +"uniqCombined64ArgMax" +"uniqCombined64ArgMin" +"uniqCombined64Array" +"uniqCombined64Distinct" +"uniqCombined64ForEach" +"uniqCombined64If" +"uniqCombined64Map" +"uniqCombined64Merge" +"uniqCombined64Null" +"uniqCombined64OrDefault" +"uniqCombined64OrNull" +"uniqCombined64Resample" +"uniqCombined64SimpleState" +"uniqCombined64State" +"uniqCombinedArgMax" +"uniqCombinedArgMin" +"uniqCombinedArray" +"uniqCombinedDistinct" +"uniqCombinedForEach" +"uniqCombinedIf" +"uniqCombinedMap" +"uniqCombinedMerge" +"uniqCombinedNull" +"uniqCombinedOrDefault" +"uniqCombinedOrNull" +"uniqCombinedResample" +"uniqCombinedSimpleState" +"uniqCombinedState" +"uniqDistinct" +"uniqExact" +"uniqExactArgMax" +"uniqExactArgMin" +"uniqExactArray" +"uniqExactDistinct" +"uniqExactForEach" +"uniqExactIf" +"uniqExactMap" +"uniqExactMerge" +"uniqExactNull" +"uniqExactOrDefault" +"uniqExactOrNull" +"uniqExactResample" +"uniqExactSimpleState" +"uniqExactState" +"uniqForEach" +"uniqHLL12" +"uniqHLL12ArgMax" +"uniqHLL12ArgMin" +"uniqHLL12Array" +"uniqHLL12Distinct" +"uniqHLL12ForEach" +"uniqHLL12If" +"uniqHLL12Map" +"uniqHLL12Merge" +"uniqHLL12Null" +"uniqHLL12OrDefault" +"uniqHLL12OrNull" +"uniqHLL12Resample" +"uniqHLL12SimpleState" +"uniqHLL12State" +"uniqIf" +"uniqMap" +"uniqMerge" +"uniqNull" +"uniqOrDefault" +"uniqOrNull" +"uniqResample" +"uniqSimpleState" +"uniqState" +"uniqTheta" +"uniqThetaArgMax" +"uniqThetaArgMin" +"uniqThetaArray" +"uniqThetaDistinct" +"uniqThetaForEach" +"uniqThetaIf" +"uniqThetaIntersect" +"uniqThetaMap" +"uniqThetaMerge" +"uniqThetaNot" +"uniqThetaNull" +"uniqThetaOrDefault" +"uniqThetaOrNull" +"uniqThetaResample" +"uniqThetaSimpleState" +"uniqThetaState" +"uniqThetaUnion" +"uniqUpTo" +"uniqUpToArgMax" +"uniqUpToArgMin" +"uniqUpToArray" +"uniqUpToDistinct" +"uniqUpToForEach" +"uniqUpToIf" +"uniqUpToMap" +"uniqUpToMerge" +"uniqUpToNull" +"uniqUpToOrDefault" +"uniqUpToOrNull" +"uniqUpToResample" +"uniqUpToSimpleState" +"uniqUpToState" +"upper" +"upperUTF8" +"uptime" +"user" +"validateNestedArraySizes" +"varPop" +"varPopArgMax" +"varPopArgMin" +"varPopArray" +"varPopDistinct" +"varPopForEach" +"varPopIf" +"varPopMap" +"varPopMerge" +"varPopNull" +"varPopOrDefault" +"varPopOrNull" +"varPopResample" +"varPopSimpleState" +"varPopStable" +"varPopStableArgMax" +"varPopStableArgMin" +"varPopStableArray" +"varPopStableDistinct" +"varPopStableForEach" +"varPopStableIf" +"varPopStableMap" +"varPopStableMerge" +"varPopStableNull" +"varPopStableOrDefault" +"varPopStableOrNull" +"varPopStableResample" +"varPopStableSimpleState" +"varPopStableState" +"varPopState" +"varSamp" +"varSampArgMax" +"varSampArgMin" +"varSampArray" +"varSampDistinct" +"varSampForEach" +"varSampIf" +"varSampMap" +"varSampMerge" +"varSampNull" +"varSampOrDefault" +"varSampOrNull" +"varSampResample" +"varSampSimpleState" +"varSampStable" +"varSampStableArgMax" +"varSampStableArgMin" +"varSampStableArray" +"varSampStableDistinct" +"varSampStableForEach" +"varSampStableIf" +"varSampStableMap" +"varSampStableMerge" +"varSampStableNull" +"varSampStableOrDefault" +"varSampStableOrNull" +"varSampStableResample" +"varSampStableSimpleState" +"varSampStableState" +"varSampState" +"variantElement" +"variantType" +"vectorDifference" +"vectorSum" +"version" +"visibleWidth" +"visitParamExtractBool" +"visitParamExtractFloat" +"visitParamExtractInt" +"visitParamExtractRaw" +"visitParamExtractString" +"visitParamExtractUInt" +"visitParamHas" +"week" +"welchTTest" +"welchTTestArgMax" +"welchTTestArgMin" +"welchTTestArray" +"welchTTestDistinct" +"welchTTestForEach" +"welchTTestIf" +"welchTTestMap" +"welchTTestMerge" +"welchTTestNull" +"welchTTestOrDefault" +"welchTTestOrNull" +"welchTTestResample" +"welchTTestSimpleState" +"welchTTestState" +"widthBucket" +"width_bucket" +"windowFunnel" +"windowFunnelArgMax" +"windowFunnelArgMin" +"windowFunnelArray" +"windowFunnelDistinct" +"windowFunnelForEach" +"windowFunnelIf" +"windowFunnelMap" +"windowFunnelMerge" +"windowFunnelNull" +"windowFunnelOrDefault" +"windowFunnelOrNull" +"windowFunnelResample" +"windowFunnelSimpleState" +"windowFunnelState" +"windowID" +"wkt" +"wordShingleMinHash" +"wordShingleMinHashArg" +"wordShingleMinHashArgCaseInsensitive" +"wordShingleMinHashArgCaseInsensitiveUTF8" +"wordShingleMinHashArgUTF8" +"wordShingleMinHashCaseInsensitive" +"wordShingleMinHashCaseInsensitiveUTF8" +"wordShingleMinHashUTF8" +"wordShingleSimHash" +"wordShingleSimHashCaseInsensitive" +"wordShingleSimHashCaseInsensitiveUTF8" +"wordShingleSimHashUTF8" +"wyHash64" +"xor" +"xxHash32" +"xxHash64" +"xxh3" +"yandexConsistentHash" +"yearweek" +"yesterday" +"zookeeperSessionUptime" diff --git a/tests/fuzz/dictionaries/key_words.dict b/tests/fuzz/dictionaries/key_words.dict index 0db2d159759..db517a2382c 100644 --- a/tests/fuzz/dictionaries/key_words.dict +++ b/tests/fuzz/dictionaries/key_words.dict @@ -1,183 +1,543 @@ -# Key words (based on keywords from antlr parser) - "ADD" +"ADD COLUMN" +"ADD CONSTRAINT" +"ADD INDEX" +"ADD PROJECTION" +"ADD STATISTIC" +"ADMIN OPTION FOR" "AFTER" +"ALGORITHM" "ALIAS" "ALL" +"ALLOWED_LATENESS" "ALTER" +"ALTER COLUMN" +"ALTER DATABASE" +"ALTER LIVE VIEW" +"ALTER POLICY" +"ALTER PROFILE" +"ALTER QUOTA" +"ALTER ROLE" +"ALTER ROW POLICY" +"ALTER SETTINGS PROFILE" +"ALTER TABLE" +"ALTER TEMPORARY TABLE" +"ALTER USER" "AND" +"AND STDOUT" "ANTI" "ANY" -"ARRAY" +"APPEND" +"APPLY" +"APPLY DELETED MASK" +"ARRAY JOIN" "AS" +"ASC" "ASCENDING" "ASOF" +"ASSUME" "AST" "ASYNC" "ATTACH" +"ATTACH PART" +"ATTACH PARTITION" +"ATTACH POLICY" +"ATTACH PROFILE" +"ATTACH QUOTA" +"ATTACH ROLE" +"ATTACH ROW POLICY" +"ATTACH SETTINGS PROFILE" +"ATTACH USER" +"AUTO_INCREMENT" +"AZURE" +"BACKUP" +"BCRYPT_HASH" +"BCRYPT_PASSWORD" +"BEGIN TRANSACTION" "BETWEEN" +"BIDIRECTIONAL" "BOTH" "BY" +"CASCADE" "CASE" "CAST" +"CHANGE" +"CHANGEABLE_IN_READONLY" +"CHANGED" +"CHAR" +"CHAR VARYING" +"CHARACTER" +"CHARACTER LARGE OBJECT" +"CHARACTER VARYING" "CHECK" -"CLEAR" +"CHECK ALL TABLES" +"CHECK TABLE" +"CLEANUP" +"CLEAR COLUMN" +"CLEAR INDEX" +"CLEAR PROJECTION" +"CLEAR STATISTIC" "CLUSTER" +"CLUSTERS" +"CN" "CODEC" "COLLATE" "COLUMN" +"COLUMNS" "COMMENT" +"COMMENT COLUMN" +"COMMIT" +"COMPRESSION" +"CONST" "CONSTRAINT" "CREATE" +"CREATE POLICY" +"CREATE PROFILE" +"CREATE QUOTA" +"CREATE ROLE" +"CREATE ROW POLICY" +"CREATE SETTINGS PROFILE" +"CREATE TABLE" +"CREATE TEMPORARY TABLE" +"CREATE USER" "CROSS" "CUBE" +"CURRENT GRANTS" +"CURRENT QUOTA" +"CURRENT ROLES" +"CURRENT ROW" +"CURRENT TRANSACTION" +"CURRENTUSER" +"CURRENT_USER" +"D" "DATABASE" "DATABASES" "DATE" "DAY" +"DAYS" +"DD" "DEDUPLICATE" "DEFAULT" -"DELAY" +"DEFAULT DATABASE" +"DEFAULT ROLE" +"DEFINER" "DELETE" +"DEPENDS ON" "DESC" "DESCENDING" "DESCRIBE" "DETACH" +"DETACH PART" +"DETACH PARTITION" "DICTIONARIES" "DICTIONARY" "DISK" "DISTINCT" -"DISTRIBUTED" +"DISTINCT ON" +"DIV" +"DOUBLE_SHA1_HASH" +"DOUBLE_SHA1_PASSWORD" "DROP" +"DROP COLUMN" +"DROP CONSTRAINT" +"DROP DEFAULT" +"DROP DETACHED PART" +"DROP DETACHED PARTITION" +"DROP INDEX" +"DROP PART" +"DROP PARTITION" +"DROP PROJECTION" +"DROP STATISTIC" +"DROP TABLE" +"DROP TEMPORARY TABLE" "ELSE" +"EMPTY" +"EMPTY AS" +"ENABLED ROLES" "END" +"ENFORCED" "ENGINE" +"EPHEMERAL" +"EPHEMERAL SEQUENTIAL" +"ESTIMATE" +"EVENT" "EVENTS" +"EVERY" +"EXCEPT" +"EXCEPT DATABASE" +"EXCEPT DATABASES" +"EXCEPT TABLE" +"EXCEPT TABLES" +"EXCHANGE DICTIONARIES" +"EXCHANGE TABLES" "EXISTS" "EXPLAIN" "EXPRESSION" -"EXTRACT" -"FETCHES" +"EXTENDED" +"EXTERNAL DDL FROM" +"FALSE" +"FETCH" +"FETCH PART" +"FETCH PARTITION" +"FIELDS" +"FILE" +"FILESYSTEM CACHE" +"FILESYSTEM CACHES" +"FILTER" "FINAL" "FIRST" -"FLUSH" +"FOLLOWING" "FOR" +"FOREIGN" +"FOREIGN KEY" +"FORGET PARTITION" "FORMAT" "FREEZE" "FROM" +"FROM INFILE" +"FROM SHARD" "FULL" +"FULLTEXT" "FUNCTION" "GLOBAL" +"GLOBAL IN" +"GLOBAL NOT IN" +"GRANT" +"GRANT OPTION FOR" +"GRANTEES" "GRANULARITY" -"GROUP" +"GROUP BY" +"GROUPING SETS" +"GROUPS" +"H" +"HASH" "HAVING" +"HDFS" +"HH" "HIERARCHICAL" +"HOST" "HOUR" +"HOURS" +"HTTP" "ID" -"IF" +"IDENTIFIED" +"IF EMPTY" +"IF EXISTS" +"IF NOT EXISTS" +"IGNORE NULLS" "ILIKE" "IN" +"IN PARTITION" "INDEX" -"INF" +"INDEXES" +"INDICES" +"INHERIT" "INJECTIVE" "INNER" -"INSERT" +"INSERT INTO" +"INTERPOLATE" +"INTERSECT" "INTERVAL" -"INTO" -"IS" +"INTO OUTFILE" +"INVISIBLE" +"INVOKER" +"IP" +"IS NOT DISTINCT FROM" +"IS NOT NULL" +"IS NULL" "IS_OBJECT_ID" "JOIN" +"KERBEROS" "KEY" +"KEY BY" +"KEYED BY" +"KEYS" "KILL" +"KIND" +"LARGE OBJECT" "LAST" "LAYOUT" +"LDAP" "LEADING" "LEFT" +"LEFT ARRAY JOIN" +"LESS THAN" +"LEVEL" "LIFETIME" +"LIGHTWEIGHT" "LIKE" "LIMIT" +"LINEAR" +"LIST" "LIVE" "LOCAL" -"LOGS" +"M" +"MATCH" "MATERIALIZE" +"MATERIALIZE COLUMN" +"MATERIALIZE INDEX" +"MATERIALIZE PROJECTION" +"MATERIALIZE STATISTIC" +"MATERIALIZE TTL" "MATERIALIZED" "MAX" +"MCS" +"MEMORY" "MERGES" +"MI" +"MICROSECOND" +"MICROSECONDS" +"MILLISECOND" +"MILLISECONDS" "MIN" "MINUTE" +"MINUTES" +"MM" +"MOD" "MODIFY" +"MODIFY COLUMN" +"MODIFY COMMENT" +"MODIFY ORDER BY" +"MODIFY QUERY" +"MODIFY REFRESH" +"MODIFY SAMPLE BY" +"MODIFY SETTING" +"MODIFY SQL SECURITY" +"MODIFY TTL" "MONTH" +"MONTHS" "MOVE" +"MOVE PART" +"MOVE PARTITION" +"MS" "MUTATION" -"NAN_SQL" -"NO" +"N" +"NAME" +"NAMED COLLECTION" +"NANOSECOND" +"NANOSECONDS" +"NEXT" +"NO ACTION" +"NO DELAY" +"NO LIMITS" +"NONE" "NOT" -"NULL_SQL" +"NOT BETWEEN" +"NOT IDENTIFIED" +"NOT ILIKE" +"NOT IN" +"NOT KEYED" +"NOT LIKE" +"NOT OVERRIDABLE" +"NO_PASSWORD" +"NS" +"NULL" "NULLS" "OFFSET" "ON" -"OPTIMIZE" +"ON DELETE" +"ON UPDATE" +"ON VOLUME" +"ONLY" +"OPTIMIZE TABLE" "OR" -"ORDER" +"OR REPLACE" +"ORDER BY" "OUTER" -"OUTFILE" +"OVER" +"OVERRIDABLE" +"PART" +"PARTIAL" "PARTITION" +"PARTITION BY" +"PARTITIONS" +"PART_MOVE_TO_SHARD" +"PASTE" +"PERIODIC REFRESH" +"PERMANENTLY" +"PERMISSIVE" +"PERSISTENT" +"PERSISTENT SEQUENTIAL" +"PIPELINE" +"PLAINTEXT_PASSWORD" +"PLAN" "POPULATE" +"PRECEDING" +"PRECISION" "PREWHERE" "PRIMARY" +"PRIMARY KEY" +"PROFILE" "PROJECTION" +"PULL" +"Protobuf" +"Q" +"QQ" "QUARTER" +"QUARTERS" +"QUERY" +"QUERY TREE" +"QUOTA" +"RANDOMIZE FOR" +"RANDOMIZED" "RANGE" -"RELOAD" +"READONLY" +"REALM" +"RECOMPRESS" +"REFERENCES" +"REFRESH" +"REGEXP" "REMOVE" +"REMOVE SAMPLE BY" +"REMOVE TTL" "RENAME" +"RENAME COLUMN" +"RENAME DATABASE" +"RENAME DICTIONARY" +"RENAME TABLE" +"RENAME TO" "REPLACE" -"REPLICA" -"REPLICATED" +"REPLACE PARTITION" +"RESET SETTING" +"RESPECT NULLS" +"RESTORE" +"RESTRICT" +"RESTRICTIVE" +"RESUME" +"REVOKE" "RIGHT" +"ROLLBACK" "ROLLUP" +"ROW" +"ROWS" +"S" +"S3" +"SALT" "SAMPLE" +"SAMPLE BY" +"SCHEME" "SECOND" +"SECONDS" "SELECT" "SEMI" -"SENDS" +"SERVER" "SET" +"SET DEFAULT" +"SET DEFAULT ROLE" +"SET FAKE TIME" +"SET NULL" +"SET ROLE" +"SET ROLE DEFAULT" +"SET TRANSACTION SNAPSHOT" "SETTINGS" +"SHA256_HASH" +"SHA256_PASSWORD" "SHOW" +"SHOW ACCESS" +"SHOW CREATE" +"SHOW ENGINES" +"SHOW FUNCTIONS" +"SHOW GRANTS" +"SHOW PRIVILEGES" +"SHOW PROCESSLIST" +"SHOW SETTING" +"SIGNED" +"SIMPLE" "SOURCE" -"START" -"STOP" -"SUBSTRING" +"SPATIAL" +"SQL SECURITY" +"SQL_TSI_DAY" +"SQL_TSI_HOUR" +"SQL_TSI_MICROSECOND" +"SQL_TSI_MILLISECOND" +"SQL_TSI_MINUTE" +"SQL_TSI_MONTH" +"SQL_TSI_NANOSECOND" +"SQL_TSI_QUARTER" +"SQL_TSI_SECOND" +"SQL_TSI_WEEK" +"SQL_TSI_YEAR" +"SS" +"SSH_KEY" +"SSL_CERTIFICATE" +"STATISTIC" +"STEP" +"STORAGE" +"STRICT" +"STRICTLY_ASCENDING" +"SUBPARTITION" +"SUBPARTITION BY" +"SUBPARTITIONS" +"SUSPEND" "SYNC" "SYNTAX" "SYSTEM" "TABLE" +"TABLE OVERRIDE" "TABLES" "TEMPORARY" +"TEMPORARY TABLE" "TEST" "THEN" -"TIES" -"TIMEOUT" "TIMESTAMP" "TO" +"TO DISK" +"TO INNER UUID" +"TO SHARD" +"TO TABLE" +"TO VOLUME" "TOP" "TOTALS" +"TRACKING ONLY" "TRAILING" -"TRIM" +"TRANSACTION" +"TRIGGER" +"TRUE" "TRUNCATE" "TTL" "TYPE" +"TYPEOF" +"UNBOUNDED" +"UNDROP" +"UNFREEZE" "UNION" +"UNIQUE" +"UNSET FAKE TIME" +"UNSIGNED" "UPDATE" +"URL" "USE" "USING" "UUID" +"VALID UNTIL" "VALUES" +"VARYING" "VIEW" -"VOLUME" +"VISIBLE" "WATCH" +"WATERMARK" "WEEK" +"WEEKS" "WHEN" "WHERE" +"WINDOW" "WITH" +"WITH ADMIN OPTION" +"WITH CHECK" +"WITH FILL" +"WITH GRANT OPTION" +"WITH NAME" +"WITH REPLACE OPTION" +"WITH TIES" +"WK" +"WRITABLE" +"WW" "YEAR" +"YEARS" +"YY" +"YYYY" +"ZKPATH" +"bagexpansion" +"base_backup" +"cluster_host_ids" +"with_itemindex" From dcaa71245164289ff9d1b74fc2516950558170a8 Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Tue, 19 Mar 2024 23:43:01 +0000 Subject: [PATCH 0314/1165] disallow LowCardinality input type for JSONExtract Workaraund for a memory corruption issue https://github.com/ClickHouse/ClickHouse/issues/61562 It seems that the root cause lies not within the parser itself, but rather either with the Columns/ColumnLowCardinality or Functions/IFunction code paths. --- src/Functions/FunctionsJSON.h | 4 +--- tests/queries/0_stateless/00918_json_functions.reference | 1 + tests/queries/0_stateless/00918_json_functions.sql | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 2539fa1aeb4..53515985e39 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -348,6 +348,7 @@ public: String getName() const override { return Name::name; } bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { @@ -469,9 +470,6 @@ public: else return_type = json_return_type; - /// Top-level LowCardinality columns are processed outside JSON parser. - json_return_type = removeLowCardinality(json_return_type); - DataTypes argument_types; argument_types.reserve(arguments.size()); for (const auto & argument : arguments) diff --git a/tests/queries/0_stateless/00918_json_functions.reference b/tests/queries/0_stateless/00918_json_functions.reference index 43b15ded93d..7b725111755 100644 --- a/tests/queries/0_stateless/00918_json_functions.reference +++ b/tests/queries/0_stateless/00918_json_functions.reference @@ -69,6 +69,7 @@ hello (3333.6,'test') (3333.6333333333,'test') (3333.6333333333,'test') +\N 123456.1234 Decimal(20, 4) 123456.1234 Decimal(20, 4) 123456789012345.12 Decimal(30, 4) diff --git a/tests/queries/0_stateless/00918_json_functions.sql b/tests/queries/0_stateless/00918_json_functions.sql index e19dd17670e..61fcb21fcbd 100644 --- a/tests/queries/0_stateless/00918_json_functions.sql +++ b/tests/queries/0_stateless/00918_json_functions.sql @@ -81,6 +81,7 @@ SELECT JSONExtract('{"a":3333.6333333333333333333333, "b":"test"}', 'Tuple(a Dec SELECT JSONExtract('{"a":"3333.6333333333333333333333", "b":"test"}', 'Tuple(a Decimal(10,1), b LowCardinality(String))'); SELECT JSONExtract('{"a":3333.6333333333333333333333, "b":"test"}', 'Tuple(a Decimal(20,10), b LowCardinality(String))'); SELECT JSONExtract('{"a":"3333.6333333333333333333333", "b":"test"}', 'Tuple(a Decimal(20,10), b LowCardinality(String))'); +SELECT JSONExtract(materialize('{"string_value":null}'), materialize('string_value'), 'LowCardinality(Nullable(String))'); SELECT JSONExtract('{"a":123456.123456}', 'a', 'Decimal(20, 4)') as a, toTypeName(a); SELECT JSONExtract('{"a":"123456.123456"}', 'a', 'Decimal(20, 4)') as a, toTypeName(a); SELECT JSONExtract('{"a":"123456789012345.12"}', 'a', 'Decimal(30, 4)') as a, toTypeName(a); @@ -326,3 +327,4 @@ SELECT JSONExtract('[]', JSONExtract('0', 'UInt256'), 'UInt256'); -- { serverErr SELECT '--show error: key of map type should be String'; SELECT JSONExtract('{"a": [100.0, 200], "b": [-100, 200.0, 300]}', 'Map(Int64, Array(Float64))'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT JSONExtract(materialize(toLowCardinality('{"string_value":null}')), materialize('string_value'), 'LowCardinality(Nullable(String))'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT } From a55fabdd5d6c46ab677f081bb4ec8a7491771ca4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 00:10:22 +0100 Subject: [PATCH 0315/1165] Inhibit randomization in the test --- tests/queries/0_stateless/02998_primary_key_skip_columns.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02998_primary_key_skip_columns.sql b/tests/queries/0_stateless/02998_primary_key_skip_columns.sql index b567155ab1f..6574a82a165 100644 --- a/tests/queries/0_stateless/02998_primary_key_skip_columns.sql +++ b/tests/queries/0_stateless/02998_primary_key_skip_columns.sql @@ -1,4 +1,4 @@ --- Tags: no-asan, no-tsan, no-msan, no-ubsan +-- Tags: no-asan, no-tsan, no-msan, no-ubsan, no-random-settings DROP TABLE IF EXISTS test; From 15aff51dfcae4679238c7cf78e542e40a2f4d4c3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 00:13:26 +0100 Subject: [PATCH 0316/1165] Inhibit randomization in the test --- .../queries/0_stateless/01019_alter_materialized_view_atomic.sh | 2 +- .../0_stateless/01171_mv_select_insert_isolation_long.sh | 2 +- .../0_stateless/01459_manual_write_to_replicas_quorum.sh | 2 +- tests/queries/0_stateless/01508_partition_pruning_long.sh | 2 +- tests/queries/0_stateless/01747_system_session_log_long.sh | 2 +- .../0_stateless/02782_uniq_exact_parallel_merging_bug.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh index e50b974a48e..4bd21fcee02 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, no-debug set -e diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh index 619e212c3ae..8344bb6f426 100755 --- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh +++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-parallel, no-ordinary-database +# Tags: long, no-parallel, no-ordinary-database, no-debug # Test is too heavy, avoid parallel run in Flaky Check # shellcheck disable=SC2119 diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh index 379f83c6271..91a73471557 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: replica, no-replicated-database, no-parallel +# Tags: replica, no-replicated-database, no-parallel, no-debug, no-random-settings # Tag no-replicated-database: Fails due to additional replicas or shards set -e diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.sh b/tests/queries/0_stateless/01508_partition_pruning_long.sh index c1f2d6562ab..8e4b9547da8 100755 --- a/tests/queries/0_stateless/01508_partition_pruning_long.sh +++ b/tests/queries/0_stateless/01508_partition_pruning_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-polymorphic-parts, no-random-merge-tree-settings +# Tags: long, no-polymorphic-parts, no-random-merge-tree-settings, no-debug # Description of test result: # Test the correctness of the partition pruning diff --git a/tests/queries/0_stateless/01747_system_session_log_long.sh b/tests/queries/0_stateless/01747_system_session_log_long.sh index 3ad33c28ee1..ecddcb627b8 100755 --- a/tests/queries/0_stateless/01747_system_session_log_long.sh +++ b/tests/queries/0_stateless/01747_system_session_log_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-parallel, no-fasttest +# Tags: long, no-parallel, no-fasttest, no-debug ################################################################################################## # Verify that login, logout, and login failure events are properly stored in system.session_log diff --git a/tests/queries/0_stateless/02782_uniq_exact_parallel_merging_bug.sh b/tests/queries/0_stateless/02782_uniq_exact_parallel_merging_bug.sh index 060efe86602..6e787523540 100755 --- a/tests/queries/0_stateless/02782_uniq_exact_parallel_merging_bug.sh +++ b/tests/queries/0_stateless/02782_uniq_exact_parallel_merging_bug.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-random-settings, no-tsan, no-asan, no-ubsan, no-msan, no-parallel +# Tags: long, no-random-settings, no-tsan, no-asan, no-ubsan, no-msan, no-parallel, no-debug # shellcheck disable=SC2154 From 618c11166d5ada0b2b37a69eef534b93997faaf9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 00:53:09 +0100 Subject: [PATCH 0317/1165] Add a test for #61669 --- .../0_stateless/03015_peder1001.reference | 1 + tests/queries/0_stateless/03015_peder1001.sql | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/03015_peder1001.reference create mode 100644 tests/queries/0_stateless/03015_peder1001.sql diff --git a/tests/queries/0_stateless/03015_peder1001.reference b/tests/queries/0_stateless/03015_peder1001.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03015_peder1001.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03015_peder1001.sql b/tests/queries/0_stateless/03015_peder1001.sql new file mode 100644 index 00000000000..810503207f2 --- /dev/null +++ b/tests/queries/0_stateless/03015_peder1001.sql @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS test_data; + +CREATE TABLE test_data +( + ShipmentDate Date +) +ENGINE = Memory; + +INSERT INTO test_data (ShipmentDate) Values ('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'), ('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-01-08'); + +SELECT + toDayOfWeek(ShipmentDate) AS c +FROM test_data +WHERE c IS NOT NULL AND lowerUTF8(formatDateTime(date_add(DAY, toInt32(c) - 1, toDate('2024-01-01')), '%W')) LIKE '%m%' +GROUP BY c +ORDER BY c ASC +LIMIT 62 +OFFSET 0; + +DROP TABLE test_data; From 93dd212f6f9602e690ec3f378691ae57b29d1112 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Wed, 20 Mar 2024 23:55:25 +0000 Subject: [PATCH 0318/1165] Fix use-of-uninitialized-value in HedgedConnections --- src/Client/HedgedConnections.cpp | 1 - src/Client/HedgedConnections.h | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 9fa79dd6b77..fb4d9a6bdcc 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -67,7 +67,6 @@ HedgedConnections::HedgedConnections( } active_connection_count = connections.size(); - offsets_with_disabled_changing_replica = 0; pipeline_for_new_replicas.add([throttler_](ReplicaState & replica_) { replica_.connection->setThrottler(throttler_); }); } diff --git a/src/Client/HedgedConnections.h b/src/Client/HedgedConnections.h index 5bc274332db..7f538804e5a 100644 --- a/src/Client/HedgedConnections.h +++ b/src/Client/HedgedConnections.h @@ -178,12 +178,12 @@ private: std::queue offsets_queue; /// The current number of valid connections to the replicas of this shard. - size_t active_connection_count; + size_t active_connection_count = 0; /// We count offsets in which we can't change replica anymore, /// it's needed to cancel choosing new replicas when we /// disabled replica changing in all offsets. - size_t offsets_with_disabled_changing_replica; + size_t offsets_with_disabled_changing_replica = 0; Pipeline pipeline_for_new_replicas; From 4d6709579bef2bf23ba8d8cd9ee6e925dd257836 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 02:28:14 +0100 Subject: [PATCH 0319/1165] Fix error --- src/Analyzer/MatcherNode.cpp | 8 ++++---- src/Analyzer/MatcherNode.h | 2 +- src/Parsers/ASTColumnsTransformers.cpp | 23 +++++++++++++---------- src/Parsers/ASTColumnsTransformers.h | 2 +- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index 48b60423688..f573b83e538 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -92,19 +92,19 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, Identifiers columns_i MatcherNode::MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - String pattern_, + std::optional pattern_, ColumnTransformersNodes column_transformers_) : IQueryTreeNode(children_size) , matcher_type(matcher_type_) , qualified_identifier(qualified_identifier_) , columns_identifiers(columns_identifiers_) { - if (!pattern_.empty()) + if (pattern_) { - columns_matcher = std::make_shared(pattern_, re2::RE2::Quiet); + columns_matcher = std::make_shared(*pattern_, re2::RE2::Quiet); if (!columns_matcher->ok()) throw DB::Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern_, columns_matcher->error()); + "COLUMNS pattern {} cannot be compiled: {}", *pattern_, columns_matcher->error()); } auto column_transformers_list_node = std::make_shared(); diff --git a/src/Analyzer/MatcherNode.h b/src/Analyzer/MatcherNode.h index 9f6b7c6ce3d..d6f077e224b 100644 --- a/src/Analyzer/MatcherNode.h +++ b/src/Analyzer/MatcherNode.h @@ -147,7 +147,7 @@ private: explicit MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - String pattern_, + std::optional pattern_, ColumnTransformersNodes column_transformers_); MatcherNodeType matcher_type; diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 42c4c6ad88a..2a61892f8cc 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -180,8 +180,8 @@ void ASTColumnsExceptTransformer::formatImpl(const FormatSettings & settings, Fo (*it)->formatImpl(settings, state, frame); } - if (!pattern.empty()) - settings.ostr << quoteString(pattern); + if (pattern) + settings.ostr << quoteString(*pattern); if (children.size() > 1) settings.ostr << ")"; @@ -203,8 +203,8 @@ void ASTColumnsExceptTransformer::appendColumnName(WriteBuffer & ostr) const (*it)->appendColumnName(ostr); } - if (!pattern.empty()) - writeQuotedString(pattern, ostr); + if (pattern) + writeQuotedString(*pattern, ostr); if (children.size() > 1) writeChar(')', ostr); @@ -213,8 +213,11 @@ void ASTColumnsExceptTransformer::appendColumnName(WriteBuffer & ostr) const void ASTColumnsExceptTransformer::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const { hash_state.update(is_strict); - hash_state.update(pattern.size()); - hash_state.update(pattern); + if (pattern) + { + hash_state.update(pattern->size()); + hash_state.update(*pattern); + } IAST::updateTreeHashImpl(hash_state, ignore_aliases); } @@ -222,7 +225,7 @@ void ASTColumnsExceptTransformer::updateTreeHashImpl(SipHash & hash_state, bool void ASTColumnsExceptTransformer::transform(ASTs & nodes) const { std::set expected_columns; - if (pattern.empty()) + if (!pattern) { for (const auto & child : children) expected_columns.insert(child->as().name()); @@ -278,13 +281,13 @@ void ASTColumnsExceptTransformer::setPattern(String pattern_) std::shared_ptr ASTColumnsExceptTransformer::getMatcher() const { - if (pattern.empty()) + if (!pattern) return {}; - auto regexp = std::make_shared(pattern, re2::RE2::Quiet); + auto regexp = std::make_shared(*pattern, re2::RE2::Quiet); if (!regexp->ok()) throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + "COLUMNS pattern {} cannot be compiled: {}", *pattern, regexp->error()); return regexp; } diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index 2d13cd85819..cedf955bee2 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -84,7 +84,7 @@ public: protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - String pattern; + std::optional pattern; }; class ASTColumnsReplaceTransformer : public IASTColumnsTransformer From 3698c0c2cfb0cd5cda2c10531ca6dcfc1e9899e6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 02:46:03 +0100 Subject: [PATCH 0320/1165] Fix build --- src/Functions/FunctionsTimeWindow.cpp | 11 +++++ src/Functions/FunctionsTimeWindow.h | 62 +++------------------------ 2 files changed, 17 insertions(+), 56 deletions(-) diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index be51978499b..2b1359964e8 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -110,6 +110,17 @@ bool checkIntervalOrTimeZoneArgument(const ColumnWithTypeAndName & argument, con return true; } +enum TimeWindowFunctionName +{ + TUMBLE, + TUMBLE_START, + TUMBLE_END, + HOP, + HOP_START, + HOP_END, + WINDOW_ID +}; + template struct TimeWindowImpl { diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index 4ea5b3a9394..65cc7e9e87c 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -21,16 +21,6 @@ namespace DB * hopEnd(window_id) * hopEnd(time_attr, hop_interval, window_interval [, timezone]) */ -enum TimeWindowFunctionName -{ - TUMBLE, - TUMBLE_START, - TUMBLE_END, - HOP, - HOP_START, - HOP_END, - WINDOW_ID -}; template struct ToStartOfTransform; @@ -73,17 +63,15 @@ struct ToStartOfTransform; TRANSFORM_TIME(Second) #undef TRANSFORM_TIME -/// NOLINTBEGIN(bugprone-macro-parentheses) - #define TRANSFORM_SUBSECONDS(INTERVAL_KIND, DEF_SCALE) \ template<> \ struct ToStartOfTransform \ { \ static Int64 execute(Int64 t, UInt64 delta, const UInt32 scale) \ { \ - if (scale <= DEF_SCALE) \ + if (scale <= (DEF_SCALE)) \ { \ - auto val = t * DecimalUtils::scaleMultiplier(DEF_SCALE - scale); \ + auto val = t * DecimalUtils::scaleMultiplier((DEF_SCALE) - scale); \ if (delta == 1) \ return val; \ else \ @@ -91,7 +79,7 @@ template<> \ } \ else \ { \ - return t - (t % (delta * DecimalUtils::scaleMultiplier(scale - DEF_SCALE))) ; \ + return t - (t % (delta * DecimalUtils::scaleMultiplier(scale - (DEF_SCALE)))) ; \ } \ } \ }; @@ -145,12 +133,12 @@ template <> \ { \ static inline NO_SANITIZE_UNDEFINED Int64 execute(Int64 t, UInt64 delta, const UInt32 scale) \ { \ - if (scale < DEF_SCALE) \ + if (scale < (DEF_SCALE)) \ { \ - return t + delta * DecimalUtils::scaleMultiplier(DEF_SCALE - scale); \ + return t + delta * DecimalUtils::scaleMultiplier((DEF_SCALE) - scale); \ } \ else \ - return t + delta * DecimalUtils::scaleMultiplier(scale - DEF_SCALE); \ + return t + delta * DecimalUtils::scaleMultiplier(scale - (DEF_SCALE)); \ } \ }; ADD_SUBSECONDS(Millisecond, 3) @@ -158,42 +146,4 @@ template <> \ ADD_SUBSECONDS(Nanosecond, 9) #undef ADD_SUBSECONDS -/// NOLINTEND(bugprone-macro-parentheses) - -template -struct TimeWindowImpl -{ - static constexpr auto name = "UNKNOWN"; - - static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name); - - static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name); -}; - -template -class FunctionTimeWindow : public IFunction -{ -public: - static constexpr auto name = TimeWindowImpl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override { return name; } - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } - - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override; -}; - -using FunctionTumble = FunctionTimeWindow; -using FunctionTumbleStart = FunctionTimeWindow; -using FunctionTumbleEnd = FunctionTimeWindow; -using FunctionHop = FunctionTimeWindow; -using FunctionWindowId = FunctionTimeWindow; -using FunctionHopStart = FunctionTimeWindow; -using FunctionHopEnd = FunctionTimeWindow; - } From bfc7c3c89edd2454ee0d6d23052d9bade28ac2a7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 02:52:51 +0100 Subject: [PATCH 0321/1165] Remove clickhouse-diagnostics --- docker/packager/binary-builder/Dockerfile | 2 +- docker/packager/binary-builder/build.sh | 18 - packages/clickhouse-common-static.yaml | 2 - programs/diagnostics/.gitignore | 30 - programs/diagnostics/CONTRIBUTION.md | 49 - programs/diagnostics/Makefile | 65 - programs/diagnostics/README.md | 167 --- .../cmd/clickhouse-diagnostics/main.go | 9 - programs/diagnostics/cmd/collect.go | 159 --- programs/diagnostics/cmd/convert.go | 1 - programs/diagnostics/cmd/help.go | 124 -- programs/diagnostics/cmd/params/params.go | 281 ---- .../diagnostics/cmd/params/params_test.go | 247 ---- programs/diagnostics/cmd/root.go | 174 --- programs/diagnostics/cmd/version.go | 24 - programs/diagnostics/go.mod | 89 -- programs/diagnostics/go.sum | 992 -------------- .../internal/collectors/clickhouse/config.go | 113 -- .../collectors/clickhouse/config_test.go | 128 -- .../internal/collectors/clickhouse/db_logs.go | 108 -- .../collectors/clickhouse/db_logs_test.go | 119 -- .../internal/collectors/clickhouse/logs.go | 140 -- .../collectors/clickhouse/logs_test.go | 147 -- .../collectors/clickhouse/queries.json | 153 --- .../internal/collectors/clickhouse/summary.go | 159 --- .../collectors/clickhouse/summary_test.go | 111 -- .../internal/collectors/clickhouse/system.go | 165 --- .../collectors/clickhouse/system_test.go | 366 ----- .../collectors/clickhouse/zookeeper.go | 153 --- .../collectors/clickhouse/zookeeper_test.go | 102 -- .../internal/collectors/registry.go | 75 -- .../internal/collectors/registry_test.go | 57 - .../internal/collectors/system/command.go | 90 -- .../collectors/system/command_test.go | 107 -- .../internal/collectors/system/file.go | 100 -- .../internal/collectors/system/file_test.go | 110 -- .../internal/collectors/system/system.go | 235 ---- .../internal/collectors/system/system_test.go | 89 -- .../internal/outputs/file/simple.go | 344 ----- .../internal/outputs/file/simple_test.go | 468 ------- .../diagnostics/internal/outputs/registry.go | 67 - .../internal/outputs/registry_test.go | 45 - .../internal/outputs/terminal/report.go | 284 ---- .../internal/platform/config/models.go | 129 -- .../internal/platform/config/models_test.go | 182 --- .../internal/platform/config/utils.go | 74 - .../internal/platform/config/utils_test.go | 142 -- .../internal/platform/data/bundle.go | 27 - .../internal/platform/data/bundle_test.go | 26 - .../internal/platform/data/database.go | 88 -- .../internal/platform/data/database_test.go | 86 -- .../internal/platform/data/field.go | 8 - .../internal/platform/data/file.go | 444 ------ .../internal/platform/data/file_test.go | 263 ---- .../internal/platform/data/frame.go | 11 - .../internal/platform/data/memory.go | 35 - .../internal/platform/data/memory_test.go | 61 - .../internal/platform/data/misc.go | 27 - .../internal/platform/database/native.go | 95 -- .../internal/platform/database/native_test.go | 289 ---- .../diagnostics/internal/platform/manager.go | 49 - .../internal/platform/manager_test.go | 100 -- .../internal/platform/test/data.go | 166 --- .../diagnostics/internal/platform/test/env.go | 16 - .../internal/platform/utils/file.go | 95 -- .../internal/platform/utils/file_test.go | 134 -- .../internal/platform/utils/process.go | 49 - .../internal/platform/utils/process_test.go | 97 -- .../internal/platform/utils/slices.go | 68 - .../internal/platform/utils/slices_test.go | 64 - .../internal/platform/utils/time.go | 7 - programs/diagnostics/internal/runner.go | 115 -- programs/diagnostics/internal/runner_test.go | 130 -- .../configs/include/xml/server-include.xml | 8 - .../configs/include/xml/user-include.xml | 20 - .../configs/include/yaml/server-include.yaml | 1 - .../configs/include/yaml/user-include.yaml | 7 - .../testdata/configs/xml/config.xml | 1195 ----------------- .../configs/xml/users.d/default-password.xml | 8 - .../testdata/configs/xml/users.xml | 57 - .../testdata/configs/yaml/config.yaml | 927 ------------- .../yaml/users.d/default-password.yaml | 6 - .../testdata/configs/yaml/users.yaml | 47 - .../testdata/configs/yandex_xml/config.xml | 1167 ---------------- .../diagnostics/testdata/docker/admin.xml | 15 - .../diagnostics/testdata/docker/custom.xml | 8 - .../logs/var/logs/clickhouse-server.err.log | 10 - .../logs/var/logs/clickhouse-server.log | 10 - .../logs/var/logs/clickhouse-server.log.gz | 1 - 89 files changed, 1 insertion(+), 13031 deletions(-) delete mode 100644 programs/diagnostics/.gitignore delete mode 100644 programs/diagnostics/CONTRIBUTION.md delete mode 100644 programs/diagnostics/Makefile delete mode 100644 programs/diagnostics/README.md delete mode 100644 programs/diagnostics/cmd/clickhouse-diagnostics/main.go delete mode 100644 programs/diagnostics/cmd/collect.go delete mode 100644 programs/diagnostics/cmd/convert.go delete mode 100644 programs/diagnostics/cmd/help.go delete mode 100644 programs/diagnostics/cmd/params/params.go delete mode 100644 programs/diagnostics/cmd/params/params_test.go delete mode 100644 programs/diagnostics/cmd/root.go delete mode 100644 programs/diagnostics/cmd/version.go delete mode 100644 programs/diagnostics/go.mod delete mode 100644 programs/diagnostics/go.sum delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/config.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/config_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/db_logs.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/logs.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/logs_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/queries.json delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/summary.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/summary_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/system.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/system_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/zookeeper.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go delete mode 100644 programs/diagnostics/internal/collectors/registry.go delete mode 100644 programs/diagnostics/internal/collectors/registry_test.go delete mode 100644 programs/diagnostics/internal/collectors/system/command.go delete mode 100644 programs/diagnostics/internal/collectors/system/command_test.go delete mode 100644 programs/diagnostics/internal/collectors/system/file.go delete mode 100644 programs/diagnostics/internal/collectors/system/file_test.go delete mode 100644 programs/diagnostics/internal/collectors/system/system.go delete mode 100644 programs/diagnostics/internal/collectors/system/system_test.go delete mode 100644 programs/diagnostics/internal/outputs/file/simple.go delete mode 100644 programs/diagnostics/internal/outputs/file/simple_test.go delete mode 100644 programs/diagnostics/internal/outputs/registry.go delete mode 100644 programs/diagnostics/internal/outputs/registry_test.go delete mode 100644 programs/diagnostics/internal/outputs/terminal/report.go delete mode 100644 programs/diagnostics/internal/platform/config/models.go delete mode 100644 programs/diagnostics/internal/platform/config/models_test.go delete mode 100644 programs/diagnostics/internal/platform/config/utils.go delete mode 100644 programs/diagnostics/internal/platform/config/utils_test.go delete mode 100644 programs/diagnostics/internal/platform/data/bundle.go delete mode 100644 programs/diagnostics/internal/platform/data/bundle_test.go delete mode 100644 programs/diagnostics/internal/platform/data/database.go delete mode 100644 programs/diagnostics/internal/platform/data/database_test.go delete mode 100644 programs/diagnostics/internal/platform/data/field.go delete mode 100644 programs/diagnostics/internal/platform/data/file.go delete mode 100644 programs/diagnostics/internal/platform/data/file_test.go delete mode 100644 programs/diagnostics/internal/platform/data/frame.go delete mode 100644 programs/diagnostics/internal/platform/data/memory.go delete mode 100644 programs/diagnostics/internal/platform/data/memory_test.go delete mode 100644 programs/diagnostics/internal/platform/data/misc.go delete mode 100644 programs/diagnostics/internal/platform/database/native.go delete mode 100644 programs/diagnostics/internal/platform/database/native_test.go delete mode 100644 programs/diagnostics/internal/platform/manager.go delete mode 100644 programs/diagnostics/internal/platform/manager_test.go delete mode 100644 programs/diagnostics/internal/platform/test/data.go delete mode 100644 programs/diagnostics/internal/platform/test/env.go delete mode 100644 programs/diagnostics/internal/platform/utils/file.go delete mode 100644 programs/diagnostics/internal/platform/utils/file_test.go delete mode 100644 programs/diagnostics/internal/platform/utils/process.go delete mode 100644 programs/diagnostics/internal/platform/utils/process_test.go delete mode 100644 programs/diagnostics/internal/platform/utils/slices.go delete mode 100644 programs/diagnostics/internal/platform/utils/slices_test.go delete mode 100644 programs/diagnostics/internal/platform/utils/time.go delete mode 100644 programs/diagnostics/internal/runner.go delete mode 100644 programs/diagnostics/internal/runner_test.go delete mode 100644 programs/diagnostics/testdata/configs/include/xml/server-include.xml delete mode 100644 programs/diagnostics/testdata/configs/include/xml/user-include.xml delete mode 100644 programs/diagnostics/testdata/configs/include/yaml/server-include.yaml delete mode 100644 programs/diagnostics/testdata/configs/include/yaml/user-include.yaml delete mode 100644 programs/diagnostics/testdata/configs/xml/config.xml delete mode 100644 programs/diagnostics/testdata/configs/xml/users.d/default-password.xml delete mode 100644 programs/diagnostics/testdata/configs/xml/users.xml delete mode 100644 programs/diagnostics/testdata/configs/yaml/config.yaml delete mode 100644 programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml delete mode 100644 programs/diagnostics/testdata/configs/yaml/users.yaml delete mode 100644 programs/diagnostics/testdata/configs/yandex_xml/config.xml delete mode 100644 programs/diagnostics/testdata/docker/admin.xml delete mode 100644 programs/diagnostics/testdata/docker/custom.xml delete mode 100644 programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log delete mode 100644 programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log delete mode 100644 programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz diff --git a/docker/packager/binary-builder/Dockerfile b/docker/packager/binary-builder/Dockerfile index c9442accd7e..73ec4275f12 100644 --- a/docker/packager/binary-builder/Dockerfile +++ b/docker/packager/binary-builder/Dockerfile @@ -61,7 +61,7 @@ RUN arch=${TARGETARCH:-amd64} \ && rm /tmp/nfpm.deb ARG GO_VERSION=1.19.10 -# We need go for clickhouse-diagnostics +# We needed go for clickhouse-diagnostics (it is not used anymore) RUN arch=${TARGETARCH:-amd64} \ && curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \ && tar -xzf /tmp/go.tgz -C /usr/local/ \ diff --git a/docker/packager/binary-builder/build.sh b/docker/packager/binary-builder/build.sh index b63643419fe..032aceb0af3 100755 --- a/docker/packager/binary-builder/build.sh +++ b/docker/packager/binary-builder/build.sh @@ -36,22 +36,6 @@ rm -f CMakeCache.txt if [ -n "$MAKE_DEB" ]; then rm -rf /build/packages/root - # NOTE: this is for backward compatibility with previous releases, - # that does not diagnostics tool (only script). - if [ -d /build/programs/diagnostics ]; then - if [ -z "$SANITIZER" ]; then - # We need to check if clickhouse-diagnostics is fine and build it - ( - cd /build/programs/diagnostics - make test-no-docker - GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build - mv clickhouse-diagnostics .. - ) - else - echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics - chmod +x /build/programs/clickhouse-diagnostics - fi - fi fi @@ -121,8 +105,6 @@ if [ -n "$MAKE_DEB" ]; then # No quotes because I want it to expand to nothing if empty. # shellcheck disable=SC2086 DESTDIR=/build/packages/root ninja $NINJA_FLAGS programs/install - cp /build/programs/clickhouse-diagnostics /build/packages/root/usr/bin - cp /build/programs/clickhouse-diagnostics /output bash -x /build/packages/build fi diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml index 238126f95fd..383ad39591c 100644 --- a/packages/clickhouse-common-static.yaml +++ b/packages/clickhouse-common-static.yaml @@ -34,8 +34,6 @@ suggests: contents: - src: root/usr/bin/clickhouse dst: /usr/bin/clickhouse -- src: root/usr/bin/clickhouse-diagnostics - dst: /usr/bin/clickhouse-diagnostics - src: root/usr/bin/clickhouse-extract-from-config dst: /usr/bin/clickhouse-extract-from-config - src: root/usr/bin/clickhouse-library-bridge diff --git a/programs/diagnostics/.gitignore b/programs/diagnostics/.gitignore deleted file mode 100644 index 5e0b0165f38..00000000000 --- a/programs/diagnostics/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -# If you prefer the allow list template instead of the deny list, see community template: -# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file -go.work - -.idea -clickhouse-diagnostics -output -vendor -bin -profile.cov -clickhouse-diagnostics.yml -dist/ diff --git a/programs/diagnostics/CONTRIBUTION.md b/programs/diagnostics/CONTRIBUTION.md deleted file mode 100644 index 00fb073cefe..00000000000 --- a/programs/diagnostics/CONTRIBUTION.md +++ /dev/null @@ -1,49 +0,0 @@ -# Contribution - -We keep things simple. Execute all commands in this folder. - -## Requirements - -- docker - tested on version 20.10.12. -- golang >= go1.17.6 - -## Building - -Creates a binary `clickhouse-diagnostics` in the local folder. Build will be versioned according to a timestamp. For a versioned release see [Releasing](#releasing). - -```bash -make build -``` - -## Linting - -We use [golangci-lint](https://golangci-lint.run/). We use a container to run so no need to install. - -```bash -make lint-go -``` - -## Running Tests - -```bash -make test -``` - -For a coverage report, - -```bash -make test-coverage -``` - -## Adding Collectors - -TODO - - -## Adding Outputs - -TODO - -## Frames - -## Parameter Types diff --git a/programs/diagnostics/Makefile b/programs/diagnostics/Makefile deleted file mode 100644 index 2e85002b871..00000000000 --- a/programs/diagnostics/Makefile +++ /dev/null @@ -1,65 +0,0 @@ -GOCMD=go -GOTEST=$(GOCMD) test -BINARY_NAME=clickhouse-diagnostics -BUILD_DIR=dist - -TIMESTAMP := $(shell date +%Y%m%d-%H%M) -COMMIT := $(shell git rev-parse --short HEAD) -MODULE := github.com/ClickHouse/ClickHouse/programs/diagnostics -VERSION := v.dev-${TIMESTAMP} -DEVLDFLAGS = -ldflags "-X ${MODULE}/cmd.Version=${VERSION} -X ${MODULE}/cmd.Commit=${COMMIT}" - -# override with env variable to test other versions e.g. 21.11.10.1 -CLICKHOUSE_VERSION ?= latest - -GREEN := $(shell tput -Txterm setaf 2) -YELLOW := $(shell tput -Txterm setaf 3) -WHITE := $(shell tput -Txterm setaf 7) -CYAN := $(shell tput -Txterm setaf 6) -RESET := $(shell tput -Txterm sgr0) - -.PHONY: all test build vendor release lint-go test-coverages dep - -all: help - -release: ## Release is delegated to goreleaser - $(shell goreleaser release --rm-dist) - -## Build: -build: ## Build a binary for local use - # timestamped version - $(GOCMD) build ${DEVLDFLAGS} -o $(BINARY_NAME) ./cmd/clickhouse-diagnostics - -clean: ## Remove build related file - rm ${BINARY_NAME} - rm -f checkstyle-report.xml ./coverage.xml ./profile.cov - -vendor: ## Copy of all packages needed to support builds and tests in the vendor directory - $(GOCMD) mod vendor - -test: ## Run the tests of the project - CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -v -race `go list ./... | grep -v ./internal/platform/test` - -test-no-docker: ## Don't run tests depending on dockerd - CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -v -race -tags no_docker `go list ./... | grep -v ./internal/platform/test` - -lint-go: ## Use golintci-lint - docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:latest-alpine golangci-lint run - -test-coverage: ## Run the tests of the project and export the coverage - CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -cover -covermode=count -coverprofile=profile.cov `go list ./... | grep -v ./internal/platform/test` - $(GOCMD) tool cover -func profile.cov - -dep: - $(shell go mod download) - -help: ## Show this help. - @echo '' - @echo 'Usage:' - @echo ' ${YELLOW}make${RESET} ${GREEN}${RESET}' - @echo '' - @echo 'Targets:' - @awk 'BEGIN {FS = ":.*?## "} { \ - if (/^[a-zA-Z_-]+:.*?##.*$$/) {printf " ${YELLOW}%-20s${GREEN}%s${RESET}\n", $$1, $$2} \ - else if (/^## .*$$/) {printf " ${CYAN}%s${RESET}\n", substr($$1,4)} \ - }' $(MAKEFILE_LIST) diff --git a/programs/diagnostics/README.md b/programs/diagnostics/README.md deleted file mode 100644 index f800bb0648e..00000000000 --- a/programs/diagnostics/README.md +++ /dev/null @@ -1,167 +0,0 @@ -# Clickhouse Diagnostics Tool - -## Purpose - -This tool provides a means of obtaining a diagnostic bundle from a ClickHouse instance. This bundle can be provided to your nearest ClickHouse support provider in order to assist with the diagnosis of issues. - -## Design Philosophy - -- **No local dependencies** to run. We compile to a platform-independent binary, hence Go. -- **Minimize resource overhead**. Improvements always welcome. -- **Extendable framework**. At its core, the tool provides collectors and outputs. Collectors are independent and are responsible for collecting a specific dataset e.g. system configuration. Outputs produce the diagnostic bundle in a specific format. It should be trivial to add both for contributors. See [Collectors](#collectors) and [Outputs](#outputs) for more details. -- **Convertible output formats**. Outputs produce diagnostic bundles in different formats e.g. archive, simple report etc. Where possible, it should be possible to convert between these formats. For example, an administrator may provide a bundle as an archive to their support provider who in turn wishes to visualise this as a report or even in ClickHouse itself... -- **Something is better than nothing**. Collectors execute independently. We never fail a collection because one fails - preferring to warn the user only. There are good reasons for a collector failure e.g. insufficient permissions or missing data. -- **Execute anywhere** - Ideally, this tool is executed on a ClickHouse host. Some collectors e.g. configuration file collection or system information, rely on this. However, collectors will obtain as much information remotely from the database as possible if executed remotely from the cluster - warning where collection fails. **We do currently require ClickHouse to be running, connecting over the native port**. - -We recommend reading [Permissions, Warnings & Locality](#permissions-warnings--locality). - -## Usage - -### Collection - -The `collect` command allows the collection of a diagnostic bundle. In its simplest form, assuming ClickHouse is running locally on default ports with no password: - -```bash -clickhouse-diagnostics collect -``` - -This will use the default collectors and the simple output. This output produces a timestamped archive bundle in `gz` format in a sub folder named after the host. This folder name can be controlled via the parameter `--id` or configured directly for the simple output parameter `output.simple.folder` (this allows a specific directory to be specified). - -Collectors, Outputs and ClickHouse connection credentials can be specified as shown below: - -```bash -clickhouse-diagnostics collect --password random --username default --collector=system_db,system --output=simple --id my_cluster_name -``` - -This collects the system database and host information from the cluster running locally. The archive bundle will be produced under a folder `my_cluster_name`. - -For further details, use the in built help (the commands below are equivalent): - -```bash -clickhouse-diagnostics collect --help -./clickhouse-diagnostics help collect -``` - -### Help & Finding parameters for collectors & outputs - -Collectors and outputs have their own parameters not listed under the help for the command for the `collect` command. These can be identified using the `help` command. Specifically, - -For more information about a specific collector. - -```bash -Use "clickhouse-diagnostics help --collector [collector]" -``` - -For more information about a specific output. - -```bash -Use "clickhouse-diagnostics help --output [output]" -``` - -### Convert - -Coming soon to a cluster near you... - -## Collectors - -We currently support the following collectors. A `*` indicates this collector is enabled by default: - -- `system_db*` - Collects all tables in the system database, except those which have been excluded and up to a specified row limit. -- `system*` - Collects summary OS and hardware statistics for the host. -- `config*` - Collects the ClickHouse configuration from the local filesystem. A best effort is made using process information if ClickHouse is not installed locally. `include_path` are also considered. -- `db_logs*` - Collects the ClickHouse logs directly from the database. -- `logs*` - Collects the ClickHouse logs directly from the database. -- `summary*` - Collects summary statistics on the database based on a set of known useful queries. This represents the easiest collector to extend - contributions are welcome to this set which can be found [here](https://github.com/ClickHouse/ClickHouse/blob/master/programs/diagnostics/internal/collectors/clickhouse/queries.json). -- `file` - Collects files based on glob patterns. Does not collect directories. To preview files which will be collected try, `clickhouse-diagnostics collect --collectors=file --collector.file.file_pattern= --output report` -- `command` - Collects the output of a user specified command. To preview output, `clickhouse-diagnostics collect --collectors=command --collector.command.command="" --output report` -- `zookeeper_db` - Collects information about zookeeper using the `system.zookeeper` table, recursively iterating the zookeeper tree/table. Note: changing the default parameter values can cause extremely high load to be placed on the database. Use with caution. By default, uses the glob `/clickhouse/{task_queue}/**` to match zookeeper paths and iterates to a max depth of 8. - -## Outputs - -We currently support the following outputs. The `simple` output is currently the default: - -- `simple` - Writes out the diagnostic bundle as files in a structured directory, optionally producing a compressed archive. -- `report` - Writes out the diagnostic bundle to the terminal as a simple report. Supports an ascii table format or markdown. -- `clickhouse` - **Under development**. This will allow a bundle to be stored in a cluster allowing visualization in common tooling e.g. Grafana. - -## Simple Output - -Since the `simple` output is the default we provide additional details here. -This output produces a timestamped archive by default in `gz` format under a directory created with either the hostname of the specified collection `--id`. As shown below, a specific folder can also be specified. Compression can also be disabled, leaving just the contents of the folder: - -```bash -./clickhouse-diagnostics help --output simple - -Writes out the diagnostic bundle as files in a structured directory, optionally producing a compressed archive. - -Usage: - --output=simple [flags] - -Flags: - --output.simple.directory string Directory in which to create dump. Defaults to the current directory. (default "./") - --output.simple.format string Format of exported files (default "csv") - --output.simple.skip_archive Don't compress output to an archive -``` - -The archive itself contains a folder for each collector. Each collector can potentially produce many discrete sets of data, known as frames. Each of these typically results in a single file within the collector's folder. For example, each query for the `summary` collector results in a correspondingly named file within the `summary` folder. - -## Permissions, Warnings & Locality - -Some collectors either require specific permissions for complete collection or should be executed on a ClickHouse host. We aim to collate these requirements below: - -- `system_db` - This collect aims to collect all tables in the `system` database. Some tables may fail if certain features are not enabled. Specifically,[allow_introspection_functions](https://clickhouse.com/docs/en/operations/settings/settings/#settings-allow_introspection_functions) is required to collect the `stack_traces` table. [access_management](https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting) must be set for the ClickHouse user specified for collection, to permit access to access management tables e.g. `quota_usage`. -- `db_logs`- The ClickHouse user must have access to the tables `query_log`,`query_thread_log` and `text_log`. -- `logs` - The system user under which the tool is executed must have access to the logs directory. It must therefore also be executed on the target ClickHouse server directly for this collector work. In cases where the logs directory is not a default location e.g. `/var/log/clickhouse-server` we will attempt to establish the location from the ClickHouse configuration. This requires permissions to read the configuration files - which in most cases requires specific permissions to be granted to the run user if you are not comfortable executing the tool under sudo or the `clickhouse` user. -- `summary`- This collector executes pre-recorded queries. Some of these read tables concerning access management, thus requiring the ClickHouse user to have the [access_management](https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting) permission. -- `config` - This collector reads and copies the local configuration files. It thus requires permissions to read the configuration files - which in most cases requires specific permissions to be granted to the run user if you are not comfortable executing the tool under sudo or the `clickhouse` user. - -**If a collector cannot collect specific data because of either execution location or permissions, it will log a warning to the terminal.** - -## Logging - -All logs are output to `stderr`. `stdout` is used exclusively for outputs to print information. - -## Configuration file - -In addition to supporting parameters via the command line, a configuration file can be specified via the `--config`, `-f` flag. - -By default, we look for a configuration file `clickhouse-diagnostics.yml` in the same directory as the binary. If not present, we revert to command line flags. - -**Values set via the command line values always take precedence over those in the configuration file.** - -All parameters can be set via the configuration file and can in most cases be converted to a yaml hierarchy, where periods indicate a nesting. For example, - -`--collector.system_db.row_limit=1` - -becomes - -```yaml -collector: - system_db: - row_limit: 1 -``` - -The following exceptions exist to avoid collisions: - -| Command | Parameter | Configuration File | -|---------|------------|--------------------| -| collect | output | collect.output | -| collect | collectors | collect.collectors | - -## FAQ - -1. Does the collector need root permissions? - - No. However, to read some local files e.g. configurations, the tool should be executed as the `clickhouse` user. - -2. What ClickHouse database permissions does the collector need? - - Read permissions on all system tables are required in most cases - although only specific collectors need this. [Access management permissions]((https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting)) will ensure full collection. - -3. Is any processing done on logs for anonimization purposes? - - Currently no. ClickHouse should not log sensitive information to logs e.g. passwords. - -4. Is sensitive information removed from configuration files e.g. passwords? - - Yes. We remove both passwords and hashed passwords. Please raise an issue if you require further information to be anonimized. We appreciate this is a sensitive topic. diff --git a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go deleted file mode 100644 index 0a849a9f520..00000000000 --- a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd" -) - -func main() { - cmd.Execute() -} diff --git a/programs/diagnostics/cmd/collect.go b/programs/diagnostics/cmd/collect.go deleted file mode 100644 index 503d8e41fb7..00000000000 --- a/programs/diagnostics/cmd/collect.go +++ /dev/null @@ -1,159 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "github.com/spf13/viper" -) - -var id string -var output = params.StringOptionsVar{ - Options: outputs.GetOutputNames(), - Value: "simple", -} - -// access credentials -var host string -var port uint16 -var username string -var password string - -var collectorNames = params.StringSliceOptionsVar{ - Options: collectors.GetCollectorNames(false), - Values: collectors.GetCollectorNames(true), -} - -// holds the collector params passed by the cli -var collectorParams params.ParamMap - -// holds the output params passed by the cli -var outputParams params.ParamMap - -const collectHelpTemplate = `Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} - {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} - -Aliases: - {{.NameAndAliases}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}}{{if .HasAvailableSubCommands}} - -Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} - -Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} - -Global Flags: -{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}} - -Additional help topics: - Use "{{.CommandPath}} [command] --help" for more information about a command. - Use "{{.Parent.Name}} help --collector [collector]" for more information about a specific collector. - Use "{{.Parent.Name}} help --output [output]" for more information about a specific output. -` - -func init() { - collectCmd.Flags().StringVar(&id, "id", getHostName(), "Id of diagnostic bundle") - - // access credentials - collectCmd.Flags().StringVar(&host, "host", "localhost", "ClickHouse host") - collectCmd.Flags().Uint16VarP(&port, "port", "p", 9000, "ClickHouse native port") - collectCmd.Flags().StringVarP(&username, "username", "u", "", "ClickHouse username") - collectCmd.Flags().StringVar(&password, "password", "", "ClickHouse password") - // collectors and outputs - collectCmd.Flags().VarP(&output, "output", "o", fmt.Sprintf("Output Format for the diagnostic Bundle, options: [%s]\n", strings.Join(output.Options, ","))) - collectCmd.Flags().VarP(&collectorNames, "collectors", "c", fmt.Sprintf("Collectors to use, options: [%s]\n", strings.Join(collectorNames.Options, ","))) - - collectorConfigs, err := collectors.BuildConfigurationOptions() - if err != nil { - log.Fatal().Err(err).Msg("Unable to build collector configurations") - } - collectorParams = params.NewParamMap(collectorConfigs) - - outputConfigs, err := outputs.BuildConfigurationOptions() - if err != nil { - log.Fatal().Err(err).Msg("Unable to build output configurations") - } - params.AddParamMapToCmd(collectorParams, collectCmd, "collector", true) - - outputParams = params.NewParamMap(outputConfigs) - params.AddParamMapToCmd(outputParams, collectCmd, "output", true) - - collectCmd.SetFlagErrorFunc(handleFlagErrors) - collectCmd.SetHelpTemplate(collectHelpTemplate) - rootCmd.AddCommand(collectCmd) -} - -var collectCmd = &cobra.Command{ - Use: "collect", - Short: "Collect a diagnostic bundle", - Long: `Collect a ClickHouse diagnostic bundle for a specified ClickHouse instance`, - PreRun: func(cmd *cobra.Command, args []string) { - bindFlagsToConfig(cmd) - }, - Example: fmt.Sprintf(`%s collect --username default --collector=%s --output=simple`, rootCmd.Name(), strings.Join(collectorNames.Options[:2], ",")), - Run: func(cmd *cobra.Command, args []string) { - log.Info().Msgf("executing collect command with %v collectors and %s output", collectorNames.Values, output.Value) - outputConfig := params.ConvertParamsToConfig(outputParams)[output.Value] - runConfig := internal.NewRunConfiguration(id, host, port, username, password, output.Value, outputConfig, collectorNames.Values, params.ConvertParamsToConfig(collectorParams)) - internal.Capture(runConfig) - os.Exit(0) - }, -} - -func getHostName() string { - name, err := os.Hostname() - if err != nil { - name = "clickhouse-diagnostics" - } - return name -} - -// these flags are nested under the cmd name in the config file to prevent collisions -var flagsToNest = []string{"output", "collectors"} - -// this saves us binding each command manually to viper -func bindFlagsToConfig(cmd *cobra.Command) { - cmd.Flags().VisitAll(func(f *pflag.Flag) { - err := viper.BindEnv(f.Name, fmt.Sprintf("%s_%s", envPrefix, - strings.ToUpper(strings.Replace(f.Name, ".", "_", -1)))) - if err != nil { - log.Error().Msgf("Unable to bind %s to config", f.Name) - } - configFlagName := f.Name - if utils.Contains(flagsToNest, f.Name) { - configFlagName = fmt.Sprintf("%s.%s", cmd.Use, configFlagName) - } - err = viper.BindPFlag(configFlagName, f) - if err != nil { - log.Error().Msgf("Unable to bind %s to config", f.Name) - } - // here we prefer the config value when the param is not set on the cmd line - if !f.Changed && viper.IsSet(configFlagName) { - val := viper.Get(configFlagName) - log.Debug().Msgf("Setting parameter %s from configuration file", f.Name) - err = cmd.Flags().Set(f.Name, fmt.Sprintf("%v", val)) - if err != nil { - log.Error().Msgf("Unable to read \"%s\" value from config", f.Name) - } else { - log.Debug().Msgf("Set parameter \"%s\" from configuration", f.Name) - } - } - }) -} diff --git a/programs/diagnostics/cmd/convert.go b/programs/diagnostics/cmd/convert.go deleted file mode 100644 index 1d619dd05e2..00000000000 --- a/programs/diagnostics/cmd/convert.go +++ /dev/null @@ -1 +0,0 @@ -package cmd diff --git a/programs/diagnostics/cmd/help.go b/programs/diagnostics/cmd/help.go deleted file mode 100644 index 750576dda25..00000000000 --- a/programs/diagnostics/cmd/help.go +++ /dev/null @@ -1,124 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" -) - -var cHelp = params.StringOptionsVar{ - Options: collectors.GetCollectorNames(false), - Value: "", -} -var oHelp = params.StringOptionsVar{ - Options: outputs.GetOutputNames(), - Value: "", -} - -func init() { - helpCmd.Flags().VarP(&cHelp, "collector", "c", "Specify collector to get description of available flags") - helpCmd.Flags().VarP(&oHelp, "output", "o", "Specify output to get description of available flags") - helpCmd.SetUsageTemplate(`Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}} - -Available Commands:{{range .Parent.Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} - -Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}} - -Alternatively use "{{.CommandPath}} [command] --help" for more information about a command. -`) - helpCmd.SetFlagErrorFunc(handleFlagErrors) - -} - -var helpCmd = &cobra.Command{ - Use: "help [command]", - Short: "Help about any command, collector or output", - Long: `Help provides help for any command, collector or output in the application.`, - Example: fmt.Sprintf(`%[1]v help collect -%[1]v help --collector=config -%[1]v help --output=simple`, rootCmd.Name()), - Run: func(c *cobra.Command, args []string) { - if len(args) != 0 { - //find the command on which help is requested - cmd, _, e := c.Root().Find(args) - if cmd == nil || e != nil { - c.Printf("Unknown help topic %#q\n", args) - cobra.CheckErr(c.Root().Usage()) - } else { - cmd.InitDefaultHelpFlag() - cobra.CheckErr(cmd.Help()) - } - return - } - if cHelp.Value != "" && oHelp.Value != "" { - log.Error().Msg("Specify either --collector or --output not both") - _ = c.Help() - os.Exit(1) - } - if cHelp.Value != "" { - collector, err := collectors.GetCollectorByName(cHelp.Value) - if err != nil { - log.Fatal().Err(err).Msgf("Unable to initialize collector %s", cHelp.Value) - } - configHelp(collector.Configuration(), "collector", cHelp.Value, collector.Description()) - } else if oHelp.Value != "" { - output, err := outputs.GetOutputByName(oHelp.Value) - if err != nil { - log.Fatal().Err(err).Msgf("Unable to initialize output %s", oHelp.Value) - } - configHelp(output.Configuration(), "output", oHelp.Value, output.Description()) - } else { - _ = c.Help() - } - os.Exit(0) - }, -} - -func configHelp(conf config.Configuration, componentType, name, description string) { - - paramMap := params.NewParamMap(map[string]config.Configuration{ - name: conf, - }) - tempHelpCmd := &cobra.Command{ - Use: fmt.Sprintf("--%s=%s", componentType, name), - Short: fmt.Sprintf("Help about the %s collector", name), - Long: description, - SilenceErrors: true, - Run: func(c *cobra.Command, args []string) { - _ = c.Help() - }, - } - params.AddParamMapToCmd(paramMap, tempHelpCmd, componentType, false) - // this is workaround to hide the help flag - tempHelpCmd.Flags().BoolP("help", "h", false, "Dummy help") - tempHelpCmd.Flags().Lookup("help").Hidden = true - tempHelpCmd.SetUsageTemplate(` -{{.Long}} - -Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}} - -Flags:{{if .HasAvailableLocalFlags}} -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{else}} - - No configuration flags available -{{end}} -`) - - _ = tempHelpCmd.Execute() -} diff --git a/programs/diagnostics/cmd/params/params.go b/programs/diagnostics/cmd/params/params.go deleted file mode 100644 index c4464aab5d2..00000000000 --- a/programs/diagnostics/cmd/params/params.go +++ /dev/null @@ -1,281 +0,0 @@ -package params - -import ( - "bytes" - "encoding/csv" - "fmt" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/spf13/cobra" -) - -type cliParamType uint8 - -const ( - String cliParamType = iota - StringList - StringOptionsList - Integer - Boolean -) - -type CliParam struct { - Description string - Default interface{} - //this should always be an address to a value - as required by cobra - Value interface{} - Type cliParamType -} - -type ParamMap map[string]map[string]CliParam - -func NewParamMap(configs map[string]config.Configuration) ParamMap { - paramMap := make(ParamMap) - for name, configuration := range configs { - for _, param := range configuration.Params { - switch p := param.(type) { - case config.StringParam: - paramMap = paramMap.createStringParam(name, p) - case config.StringListParam: - paramMap = paramMap.createStringListParam(name, p) - case config.StringOptions: - paramMap = paramMap.createStringOptionsParam(name, p) - case config.IntParam: - paramMap = paramMap.createIntegerParam(name, p) - case config.BoolParam: - paramMap = paramMap.createBoolParam(name, p) - } - } - } - return paramMap -} - -func (m ParamMap) createBoolParam(rootKey string, bParam config.BoolParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value bool - param := CliParam{ - Description: bParam.Description(), - Default: bParam.Value, - Value: &value, - Type: Boolean, - } - m[rootKey][bParam.Name()] = param - return m -} - -func (m ParamMap) createStringParam(rootKey string, sParam config.StringParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value string - param := CliParam{ - Description: sParam.Description(), - Default: sParam.Value, - Value: &value, - Type: String, - } - m[rootKey][sParam.Name()] = param - return m -} - -func (m ParamMap) createStringListParam(rootKey string, lParam config.StringListParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value []string - param := CliParam{ - Description: lParam.Description(), - Default: lParam.Values, - Value: &value, - Type: StringList, - } - m[rootKey][lParam.Name()] = param - return m -} - -func (m ParamMap) createStringOptionsParam(rootKey string, oParam config.StringOptions) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - value := StringOptionsVar{ - Options: oParam.Options, - Value: oParam.Value, - } - param := CliParam{ - Description: oParam.Description(), - Default: oParam.Value, - Value: &value, - Type: StringOptionsList, - } - m[rootKey][oParam.Name()] = param - return m -} - -func (m ParamMap) createIntegerParam(rootKey string, iParam config.IntParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value int64 - param := CliParam{ - Description: iParam.Description(), - Default: iParam.Value, - Value: &value, - Type: Integer, - } - m[rootKey][iParam.Name()] = param - return m -} - -func (c CliParam) GetConfigParam(name string) config.ConfigParam { - // this is a config being passed to a collector - required can be false - param := config.NewParam(name, c.Description, false) - switch c.Type { - case String: - return config.StringParam{ - Param: param, - // values will be pointers - Value: *(c.Value.(*string)), - } - case StringList: - return config.StringListParam{ - Param: param, - Values: *(c.Value.(*[]string)), - } - case StringOptionsList: - optionsVar := *(c.Value.(*StringOptionsVar)) - return config.StringOptions{ - Param: param, - Options: optionsVar.Options, - Value: optionsVar.Value, - } - case Integer: - return config.IntParam{ - Param: param, - Value: *(c.Value.(*int64)), - } - case Boolean: - return config.BoolParam{ - Param: param, - Value: *(c.Value.(*bool)), - } - } - return param -} - -type StringOptionsVar struct { - Options []string - Value string -} - -func (o StringOptionsVar) String() string { - return o.Value -} - -func (o *StringOptionsVar) Set(p string) error { - isIncluded := func(opts []string, val string) bool { - for _, opt := range opts { - if val == opt { - return true - } - } - return false - } - if !isIncluded(o.Options, p) { - return fmt.Errorf("%s is not included in options: %v", p, o.Options) - } - o.Value = p - return nil -} - -func (o *StringOptionsVar) Type() string { - return "string" -} - -type StringSliceOptionsVar struct { - Options []string - Values []string -} - -func (o StringSliceOptionsVar) String() string { - str, _ := writeAsCSV(o.Values) - return "[" + str + "]" -} - -func (o *StringSliceOptionsVar) Set(val string) error { - values, err := readAsCSV(val) - if err != nil { - return err - } - vValues := utils.Distinct(values, o.Options) - if len(vValues) > 0 { - return fmt.Errorf("%v are not included in options: %v", vValues, o.Options) - } - o.Values = values - return nil -} - -func (o *StringSliceOptionsVar) Type() string { - return "stringSlice" -} - -func writeAsCSV(vals []string) (string, error) { - b := &bytes.Buffer{} - w := csv.NewWriter(b) - err := w.Write(vals) - if err != nil { - return "", err - } - w.Flush() - return strings.TrimSuffix(b.String(), "\n"), nil -} - -func readAsCSV(val string) ([]string, error) { - if val == "" { - return []string{}, nil - } - stringReader := strings.NewReader(val) - csvReader := csv.NewReader(stringReader) - return csvReader.Read() -} - -func AddParamMapToCmd(paramMap ParamMap, cmd *cobra.Command, prefix string, hide bool) { - for rootKey, childMap := range paramMap { - for childKey, value := range childMap { - paramName := fmt.Sprintf("%s.%s.%s", prefix, rootKey, childKey) - switch value.Type { - case String: - cmd.Flags().StringVar(value.Value.(*string), paramName, value.Default.(string), value.Description) - case StringList: - cmd.Flags().StringSliceVar(value.Value.(*[]string), paramName, value.Default.([]string), value.Description) - case StringOptionsList: - cmd.Flags().Var(value.Value.(*StringOptionsVar), paramName, value.Description) - case Integer: - cmd.Flags().Int64Var(value.Value.(*int64), paramName, value.Default.(int64), value.Description) - case Boolean: - cmd.Flags().BoolVar(value.Value.(*bool), paramName, value.Default.(bool), value.Description) - } - // this ensures flags from collectors and outputs are not shown as they will pollute the output - if hide { - _ = cmd.Flags().MarkHidden(paramName) - } - } - } -} - -func ConvertParamsToConfig(paramMap ParamMap) map[string]config.Configuration { - configuration := make(map[string]config.Configuration) - for rootKey, childMap := range paramMap { - if _, ok := configuration[rootKey]; !ok { - configuration[rootKey] = config.Configuration{} - } - for childKey, value := range childMap { - configParam := value.GetConfigParam(childKey) - configuration[rootKey] = config.Configuration{Params: append(configuration[rootKey].Params, configParam)} - } - } - return configuration -} diff --git a/programs/diagnostics/cmd/params/params_test.go b/programs/diagnostics/cmd/params/params_test.go deleted file mode 100644 index 7671506ba59..00000000000 --- a/programs/diagnostics/cmd/params/params_test.go +++ /dev/null @@ -1,247 +0,0 @@ -package params_test - -import ( - "os" - "sort" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/spf13/cobra" - "github.com/stretchr/testify/require" -) - -var conf = map[string]config.Configuration{ - "config": { - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("directory", "A directory", false), - AllowEmpty: true, - }, - }, - }, - "system": { - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Include tables", false), - }, - config.StringListParam{ - Values: []string{"distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper"}, - Param: config.NewParam("exclude_tables", "Excluded tables", false), - }, - config.IntParam{ - Value: 100000, - Param: config.NewParam("row_limit", "Max rows", false), - }, - }, - }, - "reader": { - Params: []config.ConfigParam{ - config.StringOptions{ - Value: "csv", - Options: []string{"csv"}, - Param: config.NewParam("format", "Format of imported files", false), - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("collect_archives", "Collect archives", false), - }, - }, - }, -} - -func TestNewParamMap(t *testing.T) { - // test each of the types via NewParamMap - one with each type. the keys here can represent anything e.g. a collector name - t.Run("test param map correctly converts types", func(t *testing.T) { - paramMap := params.NewParamMap(conf) - require.Len(t, paramMap, 3) - // check config - require.Contains(t, paramMap, "config") - require.Len(t, paramMap["config"], 1) - require.Contains(t, paramMap["config"], "directory") - require.IsType(t, params.CliParam{}, paramMap["config"]["directory"]) - require.Equal(t, "A directory", paramMap["config"]["directory"].Description) - require.Equal(t, "", *(paramMap["config"]["directory"].Value.(*string))) - require.Equal(t, "", paramMap["config"]["directory"].Default) - require.Equal(t, params.String, paramMap["config"]["directory"].Type) - // check system - require.Contains(t, paramMap, "system") - require.Len(t, paramMap["system"], 3) - require.IsType(t, params.CliParam{}, paramMap["system"]["include_tables"]) - - require.Equal(t, "Include tables", paramMap["system"]["include_tables"].Description) - var value []string - require.Equal(t, &value, paramMap["system"]["include_tables"].Value) - require.Equal(t, value, paramMap["system"]["include_tables"].Default) - require.Equal(t, params.StringList, paramMap["system"]["include_tables"].Type) - - require.Equal(t, "Excluded tables", paramMap["system"]["exclude_tables"].Description) - require.IsType(t, params.CliParam{}, paramMap["system"]["exclude_tables"]) - require.Equal(t, &value, paramMap["system"]["exclude_tables"].Value) - require.Equal(t, []string{"distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper"}, paramMap["system"]["exclude_tables"].Default) - require.Equal(t, params.StringList, paramMap["system"]["exclude_tables"].Type) - - require.Equal(t, "Max rows", paramMap["system"]["row_limit"].Description) - require.IsType(t, params.CliParam{}, paramMap["system"]["row_limit"]) - var iValue int64 - require.Equal(t, &iValue, paramMap["system"]["row_limit"].Value) - require.Equal(t, int64(100000), paramMap["system"]["row_limit"].Default) - require.Equal(t, params.Integer, paramMap["system"]["row_limit"].Type) - - // check reader - require.Contains(t, paramMap, "reader") - require.Len(t, paramMap["reader"], 2) - require.IsType(t, params.CliParam{}, paramMap["reader"]["format"]) - require.Equal(t, "Format of imported files", paramMap["reader"]["format"].Description) - require.IsType(t, params.CliParam{}, paramMap["reader"]["format"]) - oValue := params.StringOptionsVar{ - Options: []string{"csv"}, - Value: "csv", - } - require.Equal(t, &oValue, paramMap["reader"]["format"].Value) - require.Equal(t, "csv", paramMap["reader"]["format"].Default) - require.Equal(t, params.StringOptionsList, paramMap["reader"]["format"].Type) - - require.IsType(t, params.CliParam{}, paramMap["reader"]["collect_archives"]) - require.Equal(t, "Collect archives", paramMap["reader"]["collect_archives"].Description) - require.IsType(t, params.CliParam{}, paramMap["reader"]["collect_archives"]) - var bVar bool - require.Equal(t, &bVar, paramMap["reader"]["collect_archives"].Value) - require.Equal(t, true, paramMap["reader"]["collect_archives"].Default) - require.Equal(t, params.Boolean, paramMap["reader"]["collect_archives"].Type) - - }) - -} - -// test GetConfigParam -func TestConvertParamsToConfig(t *testing.T) { - paramMap := params.NewParamMap(conf) - t.Run("test we can convert a param map back to a config", func(t *testing.T) { - cParam := params.ConvertParamsToConfig(paramMap) - // these will not be equal as we have some information loss e.g. allowEmpty - //require.Equal(t, conf, cParam) - // deep equality - for name := range conf { - require.Equal(t, len(conf[name].Params), len(cParam[name].Params)) - // sort both consistently - sort.Slice(conf[name].Params, func(i, j int) bool { - return conf[name].Params[i].Name() < conf[name].Params[j].Name() - }) - sort.Slice(cParam[name].Params, func(i, j int) bool { - return cParam[name].Params[i].Name() < cParam[name].Params[j].Name() - }) - for i, param := range conf[name].Params { - require.Equal(t, param.Required(), cParam[name].Params[i].Required()) - require.Equal(t, param.Name(), cParam[name].Params[i].Name()) - require.Equal(t, param.Description(), cParam[name].Params[i].Description()) - } - } - }) -} - -// create via NewParamMap and add to command AddParamMapToCmd - check contents -func TestAddParamMapToCmd(t *testing.T) { - paramMap := params.NewParamMap(conf) - t.Run("test we can add hidden params to a command", func(t *testing.T) { - testComand := &cobra.Command{ - Use: "test", - Short: "Run a test", - Long: `Longer description`, - Run: func(cmd *cobra.Command, args []string) { - os.Exit(0) - }, - } - params.AddParamMapToCmd(paramMap, testComand, "collector", true) - // check we get an error on one which doesn't exist - _, err := testComand.Flags().GetString("collector.config.random") - require.NotNil(t, err) - // check getting incorrect type - _, err = testComand.Flags().GetString("collector.system.include_tables") - require.NotNil(t, err) - - // check existence of all flags - directory, err := testComand.Flags().GetString("collector.config.directory") - require.Nil(t, err) - require.Equal(t, "", directory) - - includeTables, err := testComand.Flags().GetStringSlice("collector.system.include_tables") - require.Nil(t, err) - require.Equal(t, []string{}, includeTables) - - excludeTables, err := testComand.Flags().GetStringSlice("collector.system.exclude_tables") - require.Nil(t, err) - require.Equal(t, []string{"distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper"}, excludeTables) - - rowLimit, err := testComand.Flags().GetInt64("collector.system.row_limit") - require.Nil(t, err) - require.Equal(t, int64(100000), rowLimit) - - format, err := testComand.Flags().GetString("collector.reader.format") - require.Nil(t, err) - require.Equal(t, "csv", format) - - collectArchives, err := testComand.Flags().GetBool("collector.reader.collect_archives") - require.Nil(t, err) - require.Equal(t, true, collectArchives) - }) -} - -// test StringOptionsVar -func TestStringOptionsVar(t *testing.T) { - - t.Run("test we can set", func(t *testing.T) { - format := params.StringOptionsVar{ - Options: []string{"csv", "tsv", "native"}, - Value: "csv", - } - require.Equal(t, "csv", format.String()) - err := format.Set("tsv") - require.Nil(t, err) - require.Equal(t, "tsv", format.String()) - }) - - t.Run("test set invalid", func(t *testing.T) { - format := params.StringOptionsVar{ - Options: []string{"csv", "tsv", "native"}, - Value: "csv", - } - require.Equal(t, "csv", format.String()) - err := format.Set("random") - require.NotNil(t, err) - require.Equal(t, "random is not included in options: [csv tsv native]", err.Error()) - }) -} - -// test StringSliceOptionsVar -func TestStringSliceOptionsVar(t *testing.T) { - - t.Run("test we can set", func(t *testing.T) { - formats := params.StringSliceOptionsVar{ - Options: []string{"csv", "tsv", "native", "qsv"}, - Values: []string{"csv", "tsv"}, - } - require.Equal(t, "[csv,tsv]", formats.String()) - err := formats.Set("tsv,native") - require.Nil(t, err) - require.Equal(t, "[tsv,native]", formats.String()) - }) - - t.Run("test set invalid", func(t *testing.T) { - formats := params.StringSliceOptionsVar{ - Options: []string{"csv", "tsv", "native", "qsv"}, - Values: []string{"csv", "tsv"}, - } - require.Equal(t, "[csv,tsv]", formats.String()) - err := formats.Set("tsv,random") - require.NotNil(t, err) - require.Equal(t, "[random] are not included in options: [csv tsv native qsv]", err.Error()) - err = formats.Set("msv,random") - require.NotNil(t, err) - require.Equal(t, "[msv random] are not included in options: [csv tsv native qsv]", err.Error()) - }) - -} diff --git a/programs/diagnostics/cmd/root.go b/programs/diagnostics/cmd/root.go deleted file mode 100644 index 4cf329d5438..00000000000 --- a/programs/diagnostics/cmd/root.go +++ /dev/null @@ -1,174 +0,0 @@ -package cmd - -import ( - "fmt" - "net/http" - _ "net/http/pprof" - "os" - "strings" - "time" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func enableDebug() { - if debug { - zerolog.SetGlobalLevel(zerolog.DebugLevel) - go func() { - err := http.ListenAndServe("localhost:8080", nil) - if err != nil { - log.Error().Err(err).Msg("unable to start debugger") - } else { - log.Debug().Msg("debugger has been started on port 8080") - } - }() - } -} - -var rootCmd = &cobra.Command{ - Use: "clickhouse-diagnostics", - Short: "Capture and convert ClickHouse diagnostic bundles.", - Long: `Captures ClickHouse diagnostic bundles to a number of supported formats, including file and ClickHouse itself. Converts bundles between formats.`, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - enableDebug() - err := initializeConfig() - if err != nil { - log.Error().Err(err) - os.Exit(1) - } - return nil - }, - Example: `clickhouse-diagnostics collect`, -} - -const ( - colorRed = iota + 31 - colorGreen - colorYellow - colorMagenta = 35 - - colorBold = 1 -) - -const TimeFormat = time.RFC3339 - -var debug bool -var configFiles []string - -const ( - // The environment variable prefix of all environment variables bound to our command line flags. - // For example, --output is bound to CLICKHOUSE_DIAGNOSTIC_OUTPUT. - envPrefix = "CLICKHOUSE_DIAGNOSTIC" -) - -func init() { - rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Enable debug mode") - rootCmd.PersistentFlags().StringSliceVarP(&configFiles, "config", "f", []string{"clickhouse-diagnostics.yml", "/etc/clickhouse-diagnostics.yml"}, "Configuration file path") - // set a usage template to ensure flags on root are listed as global - rootCmd.SetUsageTemplate(`Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} - {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} - -Aliases: - {{.NameAndAliases}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}}{{if .HasAvailableSubCommands}} - -Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} - -Global Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} - -Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} - {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} - -Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}} -`) - rootCmd.SetFlagErrorFunc(handleFlagErrors) - -} - -func Execute() { - // logs go to stderr - stdout is exclusive for outputs e.g. tables - output := zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: TimeFormat} - // override the colors - output.FormatLevel = func(i interface{}) string { - var l string - if ll, ok := i.(string); ok { - switch ll { - case zerolog.LevelTraceValue: - l = colorize("TRC", colorMagenta) - case zerolog.LevelDebugValue: - l = colorize("DBG", colorMagenta) - case zerolog.LevelInfoValue: - l = colorize("INF", colorGreen) - case zerolog.LevelWarnValue: - l = colorize(colorize("WRN", colorYellow), colorBold) - case zerolog.LevelErrorValue: - l = colorize(colorize("ERR", colorRed), colorBold) - case zerolog.LevelFatalValue: - l = colorize(colorize("FTL", colorRed), colorBold) - case zerolog.LevelPanicValue: - l = colorize(colorize("PNC", colorRed), colorBold) - default: - l = colorize("???", colorBold) - } - } else { - if i == nil { - l = colorize("???", colorBold) - } else { - l = strings.ToUpper(fmt.Sprintf("%s", i))[0:3] - } - } - return l - } - output.FormatTimestamp = func(i interface{}) string { - tt := i.(string) - return colorize(tt, colorGreen) - } - log.Logger = log.Output(output) - zerolog.SetGlobalLevel(zerolog.InfoLevel) - rootCmd.SetHelpCommand(helpCmd) - if err := rootCmd.Execute(); err != nil { - log.Fatal().Err(err) - } -} - -// colorize returns the string s wrapped in ANSI code c -func colorize(s interface{}, c int) string { - return fmt.Sprintf("\x1b[%dm%v\x1b[0m", c, s) -} - -func handleFlagErrors(cmd *cobra.Command, err error) error { - fmt.Println(colorize(colorize(fmt.Sprintf("Error: %s\n", err), colorRed), colorBold)) - _ = cmd.Help() - os.Exit(1) - return nil -} - -func initializeConfig() error { - // we use the first config file we find - var configFile string - for _, confFile := range configFiles { - if ok, _ := utils.FileExists(confFile); ok { - configFile = confFile - break - } - } - if configFile == "" { - log.Warn().Msgf("config file in %s not found - config file will be ignored", configFiles) - return nil - } - viper.SetConfigFile(configFile) - if err := viper.ReadInConfig(); err != nil { - return errors.Wrapf(err, "Unable to read configuration file at %s", configFile) - } - return nil -} diff --git a/programs/diagnostics/cmd/version.go b/programs/diagnostics/cmd/version.go deleted file mode 100644 index b1c0b44171b..00000000000 --- a/programs/diagnostics/cmd/version.go +++ /dev/null @@ -1,24 +0,0 @@ -package cmd - -import ( - "fmt" - "github.com/spf13/cobra" -) - -var ( - Version = "" // set at compile time with -ldflags "-X versserv/cmd.Version=x.y.yz" - Commit = "" -) - -func init() { - rootCmd.AddCommand(versionCmd) -} - -var versionCmd = &cobra.Command{ - Use: "version", - Short: "Print the version number of clickhouse-diagnostics", - Long: `All software has versions. This is clickhouse-diagnostics`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Clickhouse Diagnostics %s (%s)\n", Version, Commit) - }, -} diff --git a/programs/diagnostics/go.mod b/programs/diagnostics/go.mod deleted file mode 100644 index 34c6b0037ae..00000000000 --- a/programs/diagnostics/go.mod +++ /dev/null @@ -1,89 +0,0 @@ -module github.com/ClickHouse/ClickHouse/programs/diagnostics - -go 1.19 - -require ( - github.com/ClickHouse/clickhouse-go/v2 v2.0.12 - github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/Masterminds/semver v1.5.0 - github.com/bmatcuk/doublestar/v4 v4.0.2 - github.com/docker/go-connections v0.4.0 - github.com/elastic/gosigar v0.14.2 - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/jaypipes/ghw v0.8.0 - github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f - github.com/mholt/archiver/v4 v4.0.0-alpha.4 - github.com/olekukonko/tablewriter v0.0.5 - github.com/pkg/errors v0.9.1 - github.com/rs/zerolog v1.26.1 - github.com/spf13/cobra v1.3.0 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.10.1 - github.com/stretchr/testify v1.8.1 - github.com/testcontainers/testcontainers-go v0.18.0 - github.com/yargevad/filepathx v1.0.0 - gopkg.in/yaml.v3 v3.0.1 -) - -require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect - github.com/andybalholm/brotli v1.0.4 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect - github.com/containerd/containerd v1.6.17 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/distribution v2.8.2+incompatible // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/docker v23.0.0+incompatible // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/dsnet/compress v0.0.1 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-ole/go-ole v1.2.4 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jaypipes/pcidb v0.6.0 // indirect - github.com/klauspost/compress v1.13.6 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect - github.com/moby/patternmatcher v0.5.0 // indirect - github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2 // indirect - github.com/opencontainers/runc v1.1.3 // indirect - github.com/paulmach/orb v0.4.0 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pierrec/lz4/v4 v4.1.14 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect - github.com/spf13/afero v1.8.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect - github.com/therootcompany/xz v1.0.1 // indirect - github.com/ulikunitz/xz v0.5.10 // indirect - go.opentelemetry.io/otel v1.4.1 // indirect - go.opentelemetry.io/otel/trace v1.4.1 // indirect - golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad // indirect - google.golang.org/grpc v1.47.0 // indirect - google.golang.org/protobuf v1.28.0 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect -) diff --git a/programs/diagnostics/go.sum b/programs/diagnostics/go.sum deleted file mode 100644 index a95dfb4fd2b..00000000000 --- a/programs/diagnostics/go.sum +++ /dev/null @@ -1,992 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/clickhouse-go v1.5.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/ClickHouse/clickhouse-go/v2 v2.0.12 h1:Nbl/NZwoM6LGJm7smNBgvtdr/rxjlIssSW3eG/Nmb9E= -github.com/ClickHouse/clickhouse-go/v2 v2.0.12/go.mod h1:u4RoNQLLM2W6hNSPYrIESLJqaWSInZVmfM+MlaAhXcg= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.9.6 h1:VwnDOgLeoi2du6dAznfmspNqTiwczvjv4K7NxuY9jsY= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= -github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= -github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.6.17 h1:XDnJIeJW0cLf6v7/+N+6L9kGrChHeXekZp2VHu6OpiY= -github.com/containerd/containerd v1.6.17/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/distribution v2.8.2+incompatible h1:k9+4DKdOG+quPFZXT/mUsiQrGu9vYCp+dXpuPkuqhk8= -github.com/distribution/distribution v2.8.2+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v23.0.0+incompatible h1:L6c28tNyqZ4/ub9AZC9d5QUuunoHHfEH4/Ue+h/E5nE= -github.com/docker/docker v23.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jaypipes/ghw v0.8.0 h1:02q1pTm9CD83vuhBsEZZhOCS128pq87uyaQeJZkp3sQ= -github.com/jaypipes/ghw v0.8.0/go.mod h1:+gR9bjm3W/HnFi90liF+Fj9GpCe/Dsibl9Im8KmC7c4= -github.com/jaypipes/pcidb v0.6.0 h1:VIM7GKVaW4qba30cvB67xSCgJPTzkG8Kzw/cbs5PHWU= -github.com/jaypipes/pcidb v0.6.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f h1:B0OD7nYl2FPQEVrw8g2uyc1lGEzNbvrKh7fspGZcbvY= -github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f/go.mod h1:aEt7p9Rvh67BYApmZwNDPpgircTO2kgdmDUoF/1QmwA= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mholt/archiver/v4 v4.0.0-alpha.4 h1:QJ4UuWgavPynEX3LXxClHDRGzYcgcvTtAMp8az7spuw= -github.com/mholt/archiver/v4 v4.0.0-alpha.4/go.mod h1:J7SYS/UTAtnO3I49RQEf+2FYZVwo7XBOh9Im43VrjNs= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= -github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= -github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f h1:J/7hjLaHLD7epG0m6TBMGmp4NQ+ibBYLfeyJWdAIFLA= -github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f/go.mod h1:15ce4BGCFxt7I5NQKT+HV0yEDxmf6fSysfEDiVo3zFM= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= -github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= -github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= -github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/paulmach/orb v0.4.0 h1:ilp1MQjRapLJ1+qcays1nZpe0mvkCY+b8JU/qBKRZ1A= -github.com/paulmach/orb v0.4.0/go.mod h1:FkcWtplUAIVqAuhAOV2d3rpbnQyliDOjOcLW9dUrfdU= -github.com/paulmach/protoscan v0.2.1-0.20210522164731-4e53c6875432/go.mod h1:2sV+uZ/oQh66m4XJVZm5iqUZ62BN88Ex1E+TTS0nLzI= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= -github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= -github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60= -github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= -github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= -github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= -github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/testcontainers/testcontainers-go v0.18.0 h1:8RXrcIQv5xX/uBOSmZd297gzvA7F0yuRA37/918o7Yg= -github.com/testcontainers/testcontainers-go v0.18.0/go.mod h1:rLC7hR2SWRjJZZNrUYiTKvUXCziNxzZiYtz9icTWYNQ= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= -github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= -github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/otel v1.4.1 h1:QbINgGDDcoQUoMJa2mMaWno49lja9sHwp6aoa2n3a4g= -go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= -go.opentelemetry.io/otel/trace v1.4.1 h1:O+16qcdTrT7zxv2J6GejTPFinSwA++cYerC5iSiF8EQ= -go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad h1:kqrS+lhvaMHCxul6sKQvKJ8nAAhlVItmZV822hYFH/U= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= -howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/programs/diagnostics/internal/collectors/clickhouse/config.go b/programs/diagnostics/internal/collectors/clickhouse/config.go deleted file mode 100644 index 92368bce6f3..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/config.go +++ /dev/null @@ -1,113 +0,0 @@ -package clickhouse - -import ( - "fmt" - "path/filepath" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" -) - -type ConfigCollector struct { - resourceManager *platform.ResourceManager -} - -func NewConfigCollector(m *platform.ResourceManager) *ConfigCollector { - return &ConfigCollector{ - resourceManager: m, - } -} - -const DefaultConfigLocation = "/etc/clickhouse-server/" -const ProcessedConfigurationLocation = "/var/lib/clickhouse/preprocessed_configs" - -func (c ConfigCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(c.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - directory, err := config.ReadStringValue(conf, "directory") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - if directory != "" { - // user has specified a directory - we therefore skip all other efforts to locate the config - frame, errs := data.NewConfigFileFrame(directory) - return &data.DiagnosticBundle{ - Frames: map[string]data.Frame{ - "user_specified": frame, - }, - Errors: data.FrameErrors{Errors: errs}, - }, nil - } - configCandidates, err := FindConfigurationFiles() - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "Unable to find configuration files") - } - frames := make(map[string]data.Frame) - var frameErrors []error - for frameName, confDir := range configCandidates { - frame, errs := data.NewConfigFileFrame(confDir) - frameErrors = append(frameErrors, errs...) - frames[frameName] = frame - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: data.FrameErrors{Errors: frameErrors}, - }, err -} - -func FindConfigurationFiles() (map[string]string, error) { - configCandidates := map[string]string{ - "default": DefaultConfigLocation, - "preprocessed": ProcessedConfigurationLocation, - } - // we don't know specifically where the config is but try to find via processes - processConfigs, err := utils.FindConfigsFromClickHouseProcesses() - if err != nil { - return nil, err - } - for i, path := range processConfigs { - confDir := filepath.Dir(path) - if len(processConfigs) == 1 { - configCandidates["process"] = confDir - break - } - configCandidates[fmt.Sprintf("process_%d", i)] = confDir - } - return configCandidates, nil -} - -func (c ConfigCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("directory", "Specify the location of the configuration files for ClickHouse Server e.g. /etc/clickhouse-server/", false), - AllowEmpty: true, - }, - }, - } -} - -func (c ConfigCollector) Description() string { - return "Collects the ClickHouse configuration from the local filesystem." -} - -func (c ConfigCollector) IsDefault() bool { - return true -} - -// here we register the collector for use -func init() { - collectors.Register("config", func() (collectors.Collector, error) { - return &ConfigCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/config_test.go b/programs/diagnostics/internal/collectors/clickhouse/config_test.go deleted file mode 100644 index 355cbb65620..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/config_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package clickhouse_test - -import ( - "encoding/xml" - "fmt" - "io" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestConfigConfiguration(t *testing.T) { - t.Run("correct configuration is returned for config collector", func(t *testing.T) { - configCollector := clickhouse.NewConfigCollector(&platform.ResourceManager{}) - conf := configCollector.Configuration() - require.Len(t, conf.Params, 1) - // check first param - require.IsType(t, config.StringParam{}, conf.Params[0]) - directory, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, directory.Required()) - require.Equal(t, directory.Name(), "directory") - require.Equal(t, "", directory.Value) - }) -} - -func TestConfigCollect(t *testing.T) { - configCollector := clickhouse.NewConfigCollector(&platform.ResourceManager{}) - - t.Run("test default file collector configuration", func(t *testing.T) { - diagSet, err := configCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - // we won't be able to collect the default configs preprocessed and default - even if clickhouse is installed - // these directories should not be readable under any permissions these tests are unrealistically executed! - // note: we may also pick up configs from a local clickhouse process - we thus allow a len >=2 but don't check this - // as its non-deterministic - require.GreaterOrEqual(t, len(diagSet.Frames), 2) - // check default key - require.Contains(t, diagSet.Frames, "default") - require.Equal(t, diagSet.Frames["default"].Name(), "/etc/clickhouse-server/") - require.Equal(t, diagSet.Frames["default"].Columns(), []string{"config"}) - // collection will have failed - checkFrame(t, diagSet.Frames["default"], nil) - // check preprocessed key - require.Contains(t, diagSet.Frames, "preprocessed") - require.Equal(t, diagSet.Frames["preprocessed"].Name(), "/var/lib/clickhouse/preprocessed_configs") - require.Equal(t, diagSet.Frames["preprocessed"].Columns(), []string{"config"}) - // min of 2 - might be more if a local installation of clickhouse is running - require.GreaterOrEqual(t, len(diagSet.Errors.Errors), 2) - }) - - t.Run("test configuration when specified", func(t *testing.T) { - // create some test files - tempDir := t.TempDir() - confDir := path.Join(tempDir, "conf") - // create an includes file - includesDir := path.Join(tempDir, "includes") - err := os.MkdirAll(includesDir, os.ModePerm) - require.Nil(t, err) - includesPath := path.Join(includesDir, "random.xml") - includesFile, err := os.Create(includesPath) - require.Nil(t, err) - xmlWriter := io.Writer(includesFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - xmlConfig := data.XmlConfig{ - XMLName: xml.Name{}, - Clickhouse: data.XmlLoggerConfig{ - XMLName: xml.Name{}, - ErrorLog: "/var/log/clickhouse-server/clickhouse-server.err.log", - Log: "/var/log/clickhouse-server/clickhouse-server.log", - }, - IncludeFrom: "", - } - err = enc.Encode(xmlConfig) - require.Nil(t, err) - // create 5 temporary config files - length is 6 for the included file - rows := make([][]interface{}, 6) - for i := 0; i < 5; i++ { - if i == 4 { - // set the includes for the last doc - xmlConfig.IncludeFrom = includesPath - } - // we want to check hierarchies are walked so create a simple folder for each file - fileDir := path.Join(confDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.xml", i)) - row := make([]interface{}, 1) - row[0] = data.XmlConfigFile{Path: filepath} - rows[i] = row - xmlFile, err := os.Create(filepath) - require.Nil(t, err) - // write a little xml so its valid - xmlConfig := xmlConfig - xmlWriter := io.Writer(xmlFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - err = enc.Encode(xmlConfig) - require.Nil(t, err) - } - diagSet, err := configCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: confDir, - Param: config.NewParam("directory", "File locations", false), - }, - }, - }) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Frames, 1) - require.Contains(t, diagSet.Frames, "user_specified") - require.Equal(t, diagSet.Frames["user_specified"].Name(), confDir) - require.Equal(t, diagSet.Frames["user_specified"].Columns(), []string{"config"}) - iConf := make([]interface{}, 1) - iConf[0] = data.XmlConfigFile{Path: includesPath, Included: true} - rows[5] = iConf - checkFrame(t, diagSet.Frames["user_specified"], rows) - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/db_logs.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs.go deleted file mode 100644 index 3253f504c1b..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/db_logs.go +++ /dev/null @@ -1,108 +0,0 @@ -package clickhouse - -import ( - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" -) - -type DBLogTable struct { - orderBy data.OrderBy - excludeColumns []string -} - -var DbLogTables = map[string]DBLogTable{ - "query_log": { - orderBy: data.OrderBy{ - Column: "event_time_microseconds", - Order: data.Asc, - }, - excludeColumns: []string{}, - }, - "query_thread_log": { - orderBy: data.OrderBy{ - Column: "event_time_microseconds", - Order: data.Asc, - }, - excludeColumns: []string{}, - }, - "text_log": { - orderBy: data.OrderBy{ - Column: "event_time_microseconds", - Order: data.Asc, - }, - excludeColumns: []string{}, - }, -} - -// This collector collects db logs - -type DBLogsCollector struct { - resourceManager *platform.ResourceManager -} - -func NewDBLogsCollector(m *platform.ResourceManager) *DBLogsCollector { - return &DBLogsCollector{ - resourceManager: m, - } -} - -func (dc *DBLogsCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(dc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - frames := make(map[string]data.Frame) - var frameErrors []error - for logTable, tableConfig := range DbLogTables { - frame, err := dc.resourceManager.DbClient.ReadTable("system", logTable, tableConfig.excludeColumns, tableConfig.orderBy, rowLimit) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to collect %s", logTable)) - } else { - frames[logTable] = frame - } - } - - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -func (dc *DBLogsCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 100000, - Param: config.NewParam("row_limit", "Maximum number of log rows to collect. Negative values mean unlimited", false), - }, - }, - } -} - -func (dc *DBLogsCollector) IsDefault() bool { - return true -} - -func (dc DBLogsCollector) Description() string { - return "Collects the ClickHouse logs directly from the database." -} - -// here we register the collector for use -func init() { - collectors.Register("db_logs", func() (collectors.Collector, error) { - return &DBLogsCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go deleted file mode 100644 index 3fc585f3352..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestDbLogsConfiguration(t *testing.T) { - t.Run("correct configuration is returned for summary collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - dbLogsCollector := clickhouse.NewDBLogsCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := dbLogsCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.IntParam{}, conf.Params[0]) - rowLimit, ok := conf.Params[0].(config.IntParam) - require.True(t, ok) - require.False(t, rowLimit.Required()) - require.Equal(t, rowLimit.Name(), "row_limit") - require.Equal(t, int64(100000), rowLimit.Value) - }) -} - -func TestDbLogsCollect(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - dbLogsCollector := clickhouse.NewDBLogsCollector(&platform.ResourceManager{ - DbClient: client, - }) - queryLogColumns := []string{"type", "event_date", "event_time", "event_time_microseconds", - "query_start_time", "query_start_time_microseconds", "query_duration_ms", "read_rows", "read_bytes", "written_rows", "written_bytes", - "result_rows", "result_bytes", "memory_usage", "current_database", "query", "formatted_query", "normalized_query_hash", - "query_kind", "databases", "tables", "columns", "projections", "views", "exception_code", "exception", "stack_trace", - "is_initial_query", "user", "query_id", "address", "port", "initial_user", "initial_query_id", "initial_address", "initial_port", - "initial_query_start_time", "initial_query_start_time_microseconds", "interface", "os_user", "client_hostname", "client_name", - "client_revision", "client_version_major", "client_version_minor", "client_version_patch", "http_method", "http_user_agent", - "http_referer", "forwarded_for", "quota_key", "revision", "log_comment", "thread_ids", "ProfileEvents", "Settings", - "used_aggregate_functions", "used_aggregate_function_combinators", "used_database_engines", "used_data_type_families", - "used_dictionaries", "used_formats", "used_functions", "used_storages", "used_table_functions"} - queryLogFrame := test.NewFakeDataFrame("queryLog", queryLogColumns, - [][]interface{}{ - {"QueryStart", "2021-12-13", "2021-12-13 12:53:20", "2021-12-13 12:53:20.590579", "2021-12-13 12:53:20", "2021-12-13 12:53:20.590579", "0", "0", "0", "0", "0", "0", "0", "0", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "2021-12-13 12:53:30", "2021-12-13 12:53:30.590579", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[]", "{}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]"}, - {"QueryFinish", "2021-12-13", "2021-12-13 12:53:30", "2021-12-13 12:53:30.607292", "2021-12-13 12:53:30", "2021-12-13 12:53:30.590579", "15", "4512", "255694", "0", "0", "4358", "173248", "4415230", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "2021-12-13 12:53:30", "2021-12-13 12:53:30.590579", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[95298,95315,95587,95316,95312,95589,95318,95586,95588,95585]", "{'Query':1,'SelectQuery':1,'ArenaAllocChunks':41,'ArenaAllocBytes':401408,'FunctionExecute':62,'NetworkSendElapsedMicroseconds':463,'NetworkSendBytes':88452,'SelectedRows':4512,'SelectedBytes':255694,'RegexpCreated':6,'ContextLock':411,'RWLockAcquiredReadLocks':190,'RealTimeMicroseconds':49221,'UserTimeMicroseconds':19811,'SystemTimeMicroseconds':2817,'SoftPageFaults':1128,'OSCPUWaitMicroseconds':127,'OSCPUVirtualTimeMicroseconds':22624,'OSWriteBytes':12288,'OSWriteChars':13312}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "['concat','notEmpty','extractAll']", "[]", "[]"}, - {"QueryStart", "2021-12-13", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "0", "0", "0", "0", "0", "0", "0", "0", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[]", "{}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]"}, - {"QueryFinish", "2021-12-13", "2021-12-13 13:02:56", "2021-12-13 13:02:56.437115", "2021-12-13 13:02:56", "2021-12-13 13:02:56.419528", "16", "4629", "258376", "0", "0", "4377", "174272", "4404694", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[95298,95318,95315,95316,95312,95588,95589,95586,95585,95587]", "{'Query':1,'SelectQuery':1,'ArenaAllocChunks':41,'ArenaAllocBytes':401408,'FunctionExecute':62,'NetworkSendElapsedMicroseconds':740,'NetworkSendBytes':88794,'SelectedRows':4629,'SelectedBytes':258376,'ContextLock':411,'RWLockAcquiredReadLocks':194,'RealTimeMicroseconds':52469,'UserTimeMicroseconds':17179,'SystemTimeMicroseconds':4218,'SoftPageFaults':569,'OSCPUWaitMicroseconds':303,'OSCPUVirtualTimeMicroseconds':25087,'OSWriteBytes':12288,'OSWriteChars':12288}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "['concat','notEmpty','extractAll']", "[]", "[]"}, - }) - - client.QueryResponses["SELECT * FROM system.query_log ORDER BY event_time_microseconds ASC LIMIT 100000"] = &queryLogFrame - - textLogColumns := []string{"event_date", "event_time", "event_time_microseconds", "microseconds", "thread_name", "thread_id", "level", "query_id", "logger_name", "message", "revision", "source_file", "source_line"} - textLogFrame := test.NewFakeDataFrame("textLog", textLogColumns, - [][]interface{}{ - {"2022-02-03", "2022-02-03 16:17:47", "2022-02-03 16:37:17.056950", "56950", "clickhouse-serv", "68947", "Information", "", "DNSCacheUpdater", "Update period 15 seconds", "54458", "../src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start()", "46"}, - {"2022-02-03", "2022-02-03 16:27:47", "2022-02-03 16:37:27.057022", "57022", "clickhouse-serv", "68947", "Information", "", "Application", "Available RAM: 62.24 GiB; physical cores: 8; logical cores: 16.", "54458", "../programs/server/Server.cpp; virtual int DB::Server::main(const std::vector &)", "1380"}, - {"2022-02-03", "2022-02-03 16:37:47", "2022-02-03 16:37:37.057484", "57484", "clickhouse-serv", "68947", "Information", "", "Application", "Listening for http://[::1]:8123", "54458", "../programs/server/Server.cpp; virtual int DB::Server::main(const std::vector &)", "1444"}, - {"2022-02-03", "2022-02-03 16:47:47", "2022-02-03 16:37:47.057527", "57527", "clickhouse-serv", "68947", "Information", "", "Application", "Listening for native protocol (tcp): [::1]:9000", "54458", "../programs/server/Server.cpp; virtual int DB::Server::main(const std::vector &)", "1444"}, - }) - - client.QueryResponses["SELECT * FROM system.text_log ORDER BY event_time_microseconds ASC LIMIT 100000"] = &textLogFrame - - // skip query_thread_log frame - often it doesn't exist anyway unless enabled - t.Run("test default db logs collection", func(t *testing.T) { - bundle, errs := dbLogsCollector.Collect(config.Configuration{}) - require.Empty(t, errs) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 2) - require.Contains(t, bundle.Frames, "text_log") - require.Contains(t, bundle.Frames, "query_log") - require.Len(t, bundle.Errors.Errors, 1) - // check query_log frame - require.Contains(t, bundle.Frames, "query_log") - require.Equal(t, queryLogColumns, bundle.Frames["query_log"].Columns()) - checkFrame(t, bundle.Frames["query_log"], queryLogFrame.Rows) - //check text_log frame - require.Contains(t, bundle.Frames, "text_log") - require.Equal(t, textLogColumns, bundle.Frames["text_log"].Columns()) - checkFrame(t, bundle.Frames["text_log"], textLogFrame.Rows) - client.Reset() - }) - - t.Run("test db logs collection with limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 1, - Param: config.NewParam("row_limit", "Maximum number of log rows to collect. Negative values mean unlimited", false), - }, - }, - } - bundle, err := dbLogsCollector.Collect(conf) - require.Empty(t, err) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 0) - require.Len(t, bundle.Errors.Errors, 3) - // populate client - client.QueryResponses["SELECT * FROM system.query_log ORDER BY event_time_microseconds ASC LIMIT 1"] = &queryLogFrame - client.QueryResponses["SELECT * FROM system.text_log ORDER BY event_time_microseconds ASC LIMIT 1"] = &textLogFrame - bundle, err = dbLogsCollector.Collect(conf) - require.Empty(t, err) - require.Len(t, bundle.Frames, 2) - require.Len(t, bundle.Errors.Errors, 1) - require.Contains(t, bundle.Frames, "text_log") - require.Contains(t, bundle.Frames, "query_log") - // check query_log frame - require.Contains(t, bundle.Frames, "query_log") - require.Equal(t, queryLogColumns, bundle.Frames["query_log"].Columns()) - checkFrame(t, bundle.Frames["query_log"], queryLogFrame.Rows[:1]) - //check text_log frame - require.Contains(t, bundle.Frames, "text_log") - require.Equal(t, textLogColumns, bundle.Frames["text_log"].Columns()) - checkFrame(t, bundle.Frames["text_log"], textLogFrame.Rows[:1]) - client.Reset() - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/logs.go b/programs/diagnostics/internal/collectors/clickhouse/logs.go deleted file mode 100644 index 8436a392c47..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/logs.go +++ /dev/null @@ -1,140 +0,0 @@ -package clickhouse - -import ( - "fmt" - "path/filepath" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" -) - -// This collector collects logs - -type LogsCollector struct { - resourceManager *platform.ResourceManager -} - -func NewLogsCollector(m *platform.ResourceManager) *LogsCollector { - return &LogsCollector{ - resourceManager: m, - } -} - -var DefaultLogsLocation = filepath.Clean("/var/log/clickhouse-server/") - -func (lc *LogsCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(lc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - directory, err := config.ReadStringValue(conf, "directory") - if err != nil { - return &data.DiagnosticBundle{}, err - } - collectArchives, err := config.ReadBoolValue(conf, "collect_archives") - if err != nil { - return &data.DiagnosticBundle{}, err - } - logPatterns := []string{"*.log"} - if collectArchives { - logPatterns = append(logPatterns, "*.gz") - } - - if directory != "" { - // user has specified a directory - we therefore skip all other efforts to locate the logs - frame, errs := data.NewFileDirectoryFrame(directory, logPatterns) - return &data.DiagnosticBundle{ - Frames: map[string]data.Frame{ - "user_specified": frame, - }, - Errors: data.FrameErrors{Errors: errs}, - }, nil - } - // add the default - frames := make(map[string]data.Frame) - dirFrame, frameErrors := data.NewFileDirectoryFrame(DefaultLogsLocation, logPatterns) - frames["default"] = dirFrame - logFolders, errs := FindLogFileCandidates() - frameErrors = append(frameErrors, errs...) - i := 0 - for folder, paths := range logFolders { - // we will collect the default location anyway above so skip these - if folder != DefaultLogsLocation { - if collectArchives { - paths = append(paths, "*.gz") - } - dirFrame, errs := data.NewFileDirectoryFrame(folder, paths) - frames[fmt.Sprintf("logs-%d", i)] = dirFrame - frameErrors = append(frameErrors, errs...) - } - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: data.FrameErrors{Errors: frameErrors}, - }, err -} - -func (lc *LogsCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - config.BoolParam{ - Param: config.NewParam("collect_archives", "Collect compressed log archive files", false), - }, - }, - } -} - -func FindLogFileCandidates() (logFolders map[string][]string, configErrors []error) { - // we need the config to determine the location of the logs - configCandidates := make(map[string]data.ConfigFileFrame) - configFiles, err := FindConfigurationFiles() - logFolders = make(map[string][]string) - if err != nil { - configErrors = append(configErrors, err) - return logFolders, configErrors - } - for _, folder := range configFiles { - configFrame, errs := data.NewConfigFileFrame(folder) - configErrors = append(configErrors, errs...) - configCandidates[filepath.Clean(folder)] = configFrame - } - - for _, config := range configCandidates { - paths, errs := config.FindLogPaths() - for _, path := range paths { - folder := filepath.Dir(path) - filename := filepath.Base(path) - if _, ok := logFolders[folder]; !ok { - logFolders[folder] = []string{} - } - logFolders[folder] = utils.Unique(append(logFolders[folder], filename)) - } - configErrors = append(configErrors, errs...) - } - return logFolders, configErrors -} - -func (lc *LogsCollector) IsDefault() bool { - return true -} - -func (lc LogsCollector) Description() string { - return "Collects the ClickHouse logs directly from the database." -} - -// here we register the collector for use -func init() { - collectors.Register("logs", func() (collectors.Collector, error) { - return &LogsCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/logs_test.go deleted file mode 100644 index 5f0be734445..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/logs_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package clickhouse_test - -import ( - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestLogsConfiguration(t *testing.T) { - t.Run("correct configuration is returned for logs collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - logsCollector := clickhouse.NewLogsCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := logsCollector.Configuration() - require.Len(t, conf.Params, 2) - // check directory - require.IsType(t, config.StringParam{}, conf.Params[0]) - directory, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, directory.Required()) - require.Equal(t, directory.Name(), "directory") - require.Empty(t, directory.Value) - // check collect_archives - require.IsType(t, config.BoolParam{}, conf.Params[1]) - collectArchives, ok := conf.Params[1].(config.BoolParam) - require.True(t, ok) - require.False(t, collectArchives.Required()) - require.Equal(t, collectArchives.Name(), "collect_archives") - require.False(t, collectArchives.Value) - }) -} - -func TestLogsCollect(t *testing.T) { - - logsCollector := clickhouse.NewLogsCollector(&platform.ResourceManager{}) - - t.Run("test default logs collection", func(t *testing.T) { - // we can't rely on a local installation of clickhouse being present for tests - if it is present (and running) - // results maybe variable e.g. we may find a config. For now, we allow flexibility and test only default. - // TODO: we may want to test this within a container - bundle, err := logsCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, bundle) - // we will have some errors if clickhouse is installed or not. If former, permission issues - if latter missing folders. - require.Greater(t, len(bundle.Errors.Errors), 0) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "default") - _, ok := bundle.Frames["default"].(data.DirectoryFileFrame) - require.True(t, ok) - // no guarantees clickhouse is installed so this bundle could have no frames - }) - - t.Run("test logs collection when directory is specified", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - logsPath := path.Join(cwd, "../../../testdata", "logs", "var", "logs") - bundle, err := logsCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: logsPath, - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - }, - }) - require.Nil(t, err) - checkDirectoryBundle(t, bundle, logsPath, []string{"clickhouse-server.log", "clickhouse-server.err.log"}) - - }) - - t.Run("test logs collection of archives", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - logsPath := path.Join(cwd, "../../../testdata", "logs", "var", "logs") - bundle, err := logsCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: logsPath, - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("collect_archives", "Collect compressed log archive files", false), - }, - }, - }) - require.Nil(t, err) - checkDirectoryBundle(t, bundle, logsPath, []string{"clickhouse-server.log", "clickhouse-server.err.log", "clickhouse-server.log.gz"}) - }) - - t.Run("test when directory does not exist", func(t *testing.T) { - tmpDir := t.TempDir() - logsPath := path.Join(tmpDir, "random") - bundle, err := logsCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: logsPath, - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - }, - }) - // not a fatal error currently - require.Nil(t, err) - require.Len(t, bundle.Errors.Errors, 1) - require.Equal(t, fmt.Sprintf("directory %s does not exist", logsPath), bundle.Errors.Errors[0].Error()) - }) -} - -func checkDirectoryBundle(t *testing.T, bundle *data.DiagnosticBundle, logsPath string, expectedFiles []string) { - require.NotNil(t, bundle) - require.Nil(t, bundle.Errors.Errors) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "user_specified") - dirFrame, ok := bundle.Frames["user_specified"].(data.DirectoryFileFrame) - require.True(t, ok) - require.Equal(t, logsPath, dirFrame.Directory) - require.Equal(t, []string{"files"}, dirFrame.Columns()) - i := 0 - fullPaths := make([]string, len(expectedFiles)) - for i, filePath := range expectedFiles { - fullPaths[i] = path.Join(logsPath, filePath) - } - for { - values, ok, err := dirFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - file, ok := values[0].(data.SimpleFile) - require.True(t, ok) - require.Contains(t, fullPaths, file.FilePath()) - i += 1 - } - require.Equal(t, len(fullPaths), i) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/queries.json b/programs/diagnostics/internal/collectors/clickhouse/queries.json deleted file mode 100644 index f5cf4362c9e..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/queries.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "queries": { - "version": [ - { - "statement": "SELECT version()" - } - ], - "databases": [ - { - "statement": "SELECT name, engine, tables, partitions, parts, formatReadableSize(bytes_on_disk) \"disk_size\" FROM system.databases db LEFT JOIN ( SELECT database, uniq(table) \"tables\", uniq(table, partition) \"partitions\", count() AS parts, sum(bytes_on_disk) \"bytes_on_disk\" FROM system.parts WHERE active GROUP BY database ) AS db_stats ON db.name = db_stats.database ORDER BY bytes_on_disk DESC LIMIT {{.Limit}}" - } - ], - "access": [ - { - "statement": "SHOW ACCESS" - } - ], - "quotas": [ - { - "statement": "SHOW QUOTA" - } - ], - "db_engines": [ - { - "statement": "SELECT engine, count() \"count\" FROM system.databases GROUP BY engine" - } - ], - "table_engines": [ - { - "statement": "SELECT engine, count() \"count\" FROM system.tables WHERE database != 'system' GROUP BY engine" - } - ], - "dictionaries": [ - { - "statement": "SELECT source, type, status, count() \"count\" FROM system.dictionaries GROUP BY source, type, status ORDER BY status DESC, source" - } - ], - "replicated_tables_by_delay": [ - { - "statement": "SELECT database, table, is_leader, is_readonly, absolute_delay, queue_size, inserts_in_queue, merges_in_queue FROM system.replicas ORDER BY absolute_delay DESC LIMIT {{.Limit}}" - } - ], - "replication_queue_by_oldest": [ - { - "statement": "SELECT database, table, replica_name, position, node_name, type, source_replica, parts_to_merge, new_part_name, create_time, required_quorum, is_detach, is_currently_executing, num_tries, last_attempt_time, last_exception, concat( 'time: ', toString(last_postpone_time), ', number: ', toString(num_postponed), ', reason: ', postpone_reason ) postpone FROM system.replication_queue ORDER BY create_time ASC LIMIT {{.Limit}}" - } - ], - "replicated_fetches": [ - { - "statement": "SELECT database, table, round(elapsed, 1) \"elapsed\", round(100 * progress, 1) \"progress\", partition_id, result_part_name, result_part_path, total_size_bytes_compressed, bytes_read_compressed, source_replica_path, source_replica_hostname, source_replica_port, interserver_scheme, to_detached, thread_id FROM system.replicated_fetches" - } - ], - "tables_by_max_partition_count": [ - { - "statement": "SELECT database, table, count() \"partitions\", sum(part_count) \"parts\", max(part_count) \"max_parts_per_partition\" FROM ( SELECT database, table, partition, count() \"part_count\" FROM system.parts WHERE active GROUP BY database, table, partition ) partitions GROUP BY database, table ORDER BY max_parts_per_partition DESC LIMIT {{.Limit}}" - } - ], - "stack_traces": [ - { - "statement": "SELECT '\\n' || arrayStringConcat( arrayMap( x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace) ), '\\n' ) AS trace FROM system.stack_trace" - } - ], - "crash_log": [ - { - "statement": "SELECT event_time, signal, thread_id, query_id, '\\n' || arrayStringConcat(trace_full, '\\n') AS trace, version FROM system.crash_log ORDER BY event_time DESC" - } - ], - "merges": [ - { - "statement": "SELECT database, table, round(elapsed, 1) \"elapsed\", round(100 * progress, 1) \"progress\", is_mutation, partition_id, result_part_path, source_part_paths, num_parts, formatReadableSize(total_size_bytes_compressed) \"total_size_compressed\", formatReadableSize(bytes_read_uncompressed) \"read_uncompressed\", formatReadableSize(bytes_written_uncompressed) \"written_uncompressed\", columns_written, formatReadableSize(memory_usage) \"memory_usage\", thread_id FROM system.merges", - "constraint": ">=20.3" - }, - { - "statement": "SELECT database, table, round(elapsed, 1) \"elapsed\", round(100 * progress, 1) \"progress\", is_mutation, partition_id, num_parts, formatReadableSize(total_size_bytes_compressed) \"total_size_compressed\", formatReadableSize(bytes_read_uncompressed) \"read_uncompressed\", formatReadableSize(bytes_written_uncompressed) \"written_uncompressed\", columns_written, formatReadableSize(memory_usage) \"memory_usage\" FROM system.merges" - } - ], - "mutations": [ - { - "statement": "SELECT database, table, mutation_id, command, create_time, parts_to_do_names, parts_to_do, is_done, latest_failed_part, latest_fail_time, latest_fail_reason FROM system.mutations WHERE NOT is_done ORDER BY create_time DESC", - "constraint": ">=20.3" - }, - { - "statement": "SELECT database, table, mutation_id, command, create_time, parts_to_do, is_done, latest_failed_part, latest_fail_time, latest_fail_reason FROM system.mutations WHERE NOT is_done ORDER BY create_time DESC" - } - ], - "recent_data_parts": [ - { - "statement": "SELECT database, table, engine, partition_id, name, part_type, active, level, disk_name, path, marks, rows, bytes_on_disk, data_compressed_bytes, data_uncompressed_bytes, marks_bytes, modification_time, remove_time, refcount, is_frozen, min_date, max_date, min_time, max_time, min_block_number, max_block_number FROM system.parts WHERE modification_time > now() - INTERVAL 3 MINUTE ORDER BY modification_time DESC", - "constraint": ">=20.3" - }, - { - "statement": "SELECT database, table, engine, partition_id, name, active, level, path, marks, rows, bytes_on_disk, data_compressed_bytes, data_uncompressed_bytes, marks_bytes, modification_time, remove_time, refcount, is_frozen, min_date, max_date, min_time, max_time, min_block_number, max_block_number FROM system.parts WHERE modification_time > now() - INTERVAL 3 MINUTE ORDER BY modification_time DESC" - } - ], - "detached_parts": [ - { - "statement": "SELECT database, table, partition_id, name, disk, reason, min_block_number, max_block_number, level FROM system.detached_parts" - } - ], - "processes": [ - { - "statement": "SELECT elapsed, query_id, normalizeQuery(query) AS normalized_query, is_cancelled, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, formatReadableSize(memory_usage) AS \"memory usage\", user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, thread_ids, ProfileEvents, Settings FROM system.processes ORDER BY elapsed DESC", - "constraint": ">=21.8" - }, - { - "statement": "SELECT elapsed, query_id, normalizeQuery(query) AS normalized_query, is_cancelled, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, formatReadableSize(memory_usage) AS \"memory usage\", user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.processes ORDER BY elapsed DESC", - "constraint": ">=21.3" - }, - { - "statement": "SELECT elapsed, query_id, normalizeQuery(query) AS normalized_query, is_cancelled, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, formatReadableSize(memory_usage) AS \"memory usage\", user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.processes ORDER BY elapsed DESC" - } - ], - "top_queries_by_duration": [ - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents, Settings FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY query_duration_ms DESC LIMIT {{.Limit}}", - "constraint": ">=21.8" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY query_duration_ms DESC LIMIT {{.Limit}}", - "constraint": ">=21.3" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY query_duration_ms DESC LIMIT {{.Limit}}" - } - ], - "top_queries_by_memory": [ - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents, Settings FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY memory_usage DESC LIMIT {{.Limit}}", - "constraint": ">=21.8" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY memory_usage DESC LIMIT {{.Limit}}", - "constraint": ">=21.3" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY memory_usage DESC LIMIT {{.Limit}}" - } - ], - "failed_queries": [ - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents, Settings FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY AND exception != '' ORDER BY query_start_time DESC LIMIT {{.Limit}}", - "constraint": ">=21.8" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY AND exception != '' ORDER BY query_start_time DESC LIMIT {{.Limit}}", - "constraint": ">=21.3" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY AND exception != '' ORDER BY query_start_time DESC LIMIT {{.Limit}}" - } - ] - } -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/summary.go b/programs/diagnostics/internal/collectors/clickhouse/summary.go deleted file mode 100644 index 0b6dd3aff20..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/summary.go +++ /dev/null @@ -1,159 +0,0 @@ -package clickhouse - -import ( - "bytes" - _ "embed" - "encoding/json" - "strings" - "text/template" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/Masterminds/semver" - "github.com/pkg/errors" -) - -// This collector collects the system db from database - -type SummaryCollector struct { - resourceManager *platform.ResourceManager -} - -type querySet struct { - Queries map[string][]query `json:"queries"` -} - -type query struct { - Statement string `json:"statement"` - Constraint string `json:"constraint"` -} - -type ParameterTemplate struct { - Limit int64 -} - -//go:embed queries.json -var queryFile []byte - -func NewSummaryCollector(m *platform.ResourceManager) *SummaryCollector { - return &SummaryCollector{ - resourceManager: m, - } -} - -func (sc *SummaryCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(sc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - var queries querySet - err = json.Unmarshal(queryFile, &queries) - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrap(err, "Unable to read queries from disk") - } - limit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - paramTemplate := ParameterTemplate{ - Limit: limit, - } - frames := make(map[string]data.Frame) - - serverVersion, err := getServerSemVersion(sc) - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "Unable to read server version") - } - - var frameErrors []error - for queryId, sqlQueries := range queries.Queries { - // we find the first matching query that satisfies the current version. Empty version means ANY version is - // supported - for _, sqlQuery := range sqlQueries { - var queryConstraint *semver.Constraints - if sqlQuery.Constraint != "" { - queryConstraint, err = semver.NewConstraint(sqlQuery.Constraint) - if err != nil { - //we try another one - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to parse version %s for query %s", sqlQuery.Constraint, queryId)) - continue - } - } - if sqlQuery.Constraint == "" || queryConstraint.Check(serverVersion) { - tmpl, err := template.New(queryId).Parse(sqlQuery.Statement) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to parse query %s", queryId)) - //we try another one - continue - } - buf := new(bytes.Buffer) - err = tmpl.Execute(buf, paramTemplate) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to process query %s template", queryId)) - //we try another one - continue - } - frame, err := sc.resourceManager.DbClient.ExecuteStatement(queryId, buf.String()) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to execute query %s", queryId)) - //we try another one - } else { - frames[queryId] = frame - // only 1 query executed - break - } - } - } - - } - - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -func getServerSemVersion(sc *SummaryCollector) (*semver.Version, error) { - serverVersion, err := sc.resourceManager.DbClient.Version() - if err != nil { - return &semver.Version{}, err - } - //drop the build number - it is not a semantic version - versionComponents := strings.Split(serverVersion, ".") - serverVersion = strings.Join(versionComponents[:len(versionComponents)-1], ".") - return semver.NewVersion(serverVersion) -} - -func (sc *SummaryCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 20, - Param: config.NewParam("row_limit", "Limit rows on supported queries.", false), - }, - }, - } -} - -func (sc *SummaryCollector) IsDefault() bool { - return true -} - -func (sc *SummaryCollector) Description() string { - return "Collects summary statistics on the database based on a set of known useful queries." -} - -// here we register the collector for use -func init() { - collectors.Register("summary", func() (collectors.Collector, error) { - return &SummaryCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/summary_test.go b/programs/diagnostics/internal/collectors/clickhouse/summary_test.go deleted file mode 100644 index 92945d987ed..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/summary_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestSummaryConfiguration(t *testing.T) { - t.Run("correct configuration is returned for summary collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - summaryCollector := clickhouse.NewSummaryCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := summaryCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.IntParam{}, conf.Params[0]) - limit, ok := conf.Params[0].(config.IntParam) - require.True(t, ok) - require.False(t, limit.Required()) - require.Equal(t, limit.Name(), "row_limit") - require.Equal(t, int64(20), limit.Value) - }) -} - -func TestSummaryCollection(t *testing.T) { - - client := test.NewFakeClickhouseClient(make(map[string][]string)) - versionFrame := test.NewFakeDataFrame("version", []string{"version()"}, - [][]interface{}{ - {"22.1.3.7"}, - }, - ) - client.QueryResponses["SELECT version()"] = &versionFrame - databasesFrame := test.NewFakeDataFrame("databases", []string{"name", "engine", "tables", "partitions", "parts", "disk_size"}, - [][]interface{}{ - {"tutorial", "Atomic", 2, 2, 2, "1.70 GiB"}, - {"default", "Atomic", 5, 5, 6, "1.08 GiB"}, - {"system", "Atomic", 11, 24, 70, "1.05 GiB"}, - {"INFORMATION_SCHEMA", "Memory", 0, 0, 0, "0.00 B"}, - {"covid19db", "Atomic", 0, 0, 0, "0.00 B"}, - {"information_schema", "Memory", 0, 0, 0, "0.00 B"}}) - - client.QueryResponses["SELECT name, engine, tables, partitions, parts, formatReadableSize(bytes_on_disk) \"disk_size\" "+ - "FROM system.databases db LEFT JOIN ( SELECT database, uniq(table) \"tables\", uniq(table, partition) \"partitions\", "+ - "count() AS parts, sum(bytes_on_disk) \"bytes_on_disk\" FROM system.parts WHERE active GROUP BY database ) AS db_stats "+ - "ON db.name = db_stats.database ORDER BY bytes_on_disk DESC LIMIT 20"] = &databasesFrame - - summaryCollector := clickhouse.NewSummaryCollector(&platform.ResourceManager{ - DbClient: client, - }) - - t.Run("test default summary collection", func(t *testing.T) { - bundle, errs := summaryCollector.Collect(config.Configuration{}) - require.Empty(t, errs) - require.Len(t, bundle.Errors.Errors, 30) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 2) - // check version frame - require.Contains(t, bundle.Frames, "version") - require.Equal(t, []string{"version()"}, bundle.Frames["version"].Columns()) - checkFrame(t, bundle.Frames["version"], versionFrame.Rows) - //check databases frame - require.Contains(t, bundle.Frames, "databases") - require.Equal(t, []string{"name", "engine", "tables", "partitions", "parts", "disk_size"}, bundle.Frames["databases"].Columns()) - checkFrame(t, bundle.Frames["databases"], databasesFrame.Rows) - client.Reset() - }) - - t.Run("test summary collection with limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 1, - Param: config.NewParam("row_limit", "Limit rows on supported queries.", false), - }, - }, - } - bundle, errs := summaryCollector.Collect(conf) - - require.Empty(t, errs) - require.Len(t, bundle.Errors.Errors, 31) - require.NotNil(t, bundle) - // databases will be absent due to limit - require.Len(t, bundle.Frames, 1) - // check version frame - require.Contains(t, bundle.Frames, "version") - require.Equal(t, []string{"version()"}, bundle.Frames["version"].Columns()) - checkFrame(t, bundle.Frames["version"], versionFrame.Rows) - - client.QueryResponses["SELECT name, engine, tables, partitions, parts, formatReadableSize(bytes_on_disk) \"disk_size\" "+ - "FROM system.databases db LEFT JOIN ( SELECT database, uniq(table) \"tables\", uniq(table, partition) \"partitions\", "+ - "count() AS parts, sum(bytes_on_disk) \"bytes_on_disk\" FROM system.parts WHERE active GROUP BY database ) AS db_stats "+ - "ON db.name = db_stats.database ORDER BY bytes_on_disk DESC LIMIT 1"] = &databasesFrame - bundle, errs = summaryCollector.Collect(conf) - require.Empty(t, errs) - require.Len(t, bundle.Errors.Errors, 30) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 2) - require.Contains(t, bundle.Frames, "version") - //check databases frame - require.Contains(t, bundle.Frames, "databases") - require.Equal(t, []string{"name", "engine", "tables", "partitions", "parts", "disk_size"}, bundle.Frames["databases"].Columns()) - // this will parse as our mock client does not read statement (specifically the limit clause) when called with execute - checkFrame(t, bundle.Frames["databases"], databasesFrame.Rows) - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/system.go b/programs/diagnostics/internal/collectors/clickhouse/system.go deleted file mode 100644 index d47cfd924f3..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/system.go +++ /dev/null @@ -1,165 +0,0 @@ -package clickhouse - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" -) - -// This collector collects the system db from database - -type SystemDatabaseCollector struct { - resourceManager *platform.ResourceManager -} - -const SystemDatabase = "system" - -// ExcludeColumns columns if we need - this will be refined over time [table_name][columnA, columnB] -var ExcludeColumns = map[string][]string{} - -// BannedTables - Hardcoded list. These are always excluded even if the user doesn't specify in exclude_tables. -//Attempts to export will work but we will warn -var BannedTables = []string{"numbers", "zeros"} - -// OrderBy contains a map of tables to an order by clause - by default we don't order table dumps -var OrderBy = map[string]data.OrderBy{ - "errors": { - Column: "last_error_message", - Order: data.Desc, - }, - "replication_queue": { - Column: "create_time", - Order: data.Asc, - }, -} - -func NewSystemDatabaseCollector(m *platform.ResourceManager) *SystemDatabaseCollector { - return &SystemDatabaseCollector{ - resourceManager: m, - } -} - -func (sc *SystemDatabaseCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(sc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - includeTables, err := config.ReadStringListValues(conf, "include_tables") - if err != nil { - return &data.DiagnosticBundle{}, err - } - excludeTables, err := config.ReadStringListValues(conf, "exclude_tables") - if err != nil { - return &data.DiagnosticBundle{}, err - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - excludeTables = checkBannedTables(includeTables, excludeTables) - ds, err := sc.readSystemAllTables(includeTables, excludeTables, rowLimit) - if err != nil { - return &data.DiagnosticBundle{}, err - } - return ds, nil -} - -// all banned tables are added to excluded if not present and not specified in included. Returns new exclude_tables list. -func checkBannedTables(includeTables []string, excludeTables []string) []string { - for _, bannedTable := range BannedTables { - //if its specified we don't add to our exclude list - explicitly included tables take precedence - if !utils.Contains(includeTables, bannedTable) && !utils.Contains(excludeTables, bannedTable) { - excludeTables = append(excludeTables, bannedTable) - } - } - return excludeTables -} - -func (sc *SystemDatabaseCollector) readSystemAllTables(include []string, exclude []string, limit int64) (*data.DiagnosticBundle, error) { - tableNames, err := sc.resourceManager.DbClient.ReadTableNamesForDatabase(SystemDatabase) - if err != nil { - return nil, err - } - var frameErrors []error - if include != nil { - // nil means include everything - tableNames = utils.Intersection(tableNames, include) - if len(tableNames) != len(include) { - // we warn that some included tables aren't present in db - frameErrors = append(frameErrors, fmt.Errorf("some tables specified in the include_tables are not in the system database and will not be exported: %v", - utils.Distinct(include, tableNames))) - } - } - - // exclude tables unless specified in includes - excludedTables := utils.Distinct(exclude, include) - tableNames = utils.Distinct(tableNames, excludedTables) - frames := make(map[string]data.Frame) - - for _, tableName := range tableNames { - var excludeColumns []string - if _, ok := ExcludeColumns[tableName]; ok { - excludeColumns = ExcludeColumns[tableName] - } - orderBy := data.OrderBy{} - if _, ok := OrderBy[tableName]; ok { - orderBy = OrderBy[tableName] - } - frame, err := sc.resourceManager.DbClient.ReadTable(SystemDatabase, tableName, excludeColumns, orderBy, limit) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to collect %s", tableName)) - } else { - frames[tableName] = frame - } - } - - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -func (sc *SystemDatabaseCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect. Takes precedence over exclude_tables. If not specified (default) all tables except exclude_tables.", false), - }, - config.StringListParam{ - Values: []string{"licenses", "distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper", "aggregate_function_combinators", "collations", "contributors", "data_type_families", "formats", "graphite_retentions", "numbers", "numbers_mt", "one", "parts_columns", "projection_parts", "projection_parts_columns", "table_engines", "time_zones", "zeros", "zeros_mt"}, - Param: config.NewParam("exclude_tables", "Specify list of tables to not collect.", false), - }, - config.IntParam{ - Value: 100000, - Param: config.NewParam("row_limit", "Maximum number of rows to collect from any table. Negative values mean unlimited.", false), - }, - }, - } -} - -func (sc *SystemDatabaseCollector) IsDefault() bool { - return true -} - -func (sc *SystemDatabaseCollector) Description() string { - return "Collects all tables in the system database, except those which have been excluded." -} - -// here we register the collector for use -func init() { - collectors.Register("system_db", func() (collectors.Collector, error) { - return &SystemDatabaseCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/system_test.go b/programs/diagnostics/internal/collectors/clickhouse/system_test.go deleted file mode 100644 index d1b9a6e7859..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/system_test.go +++ /dev/null @@ -1,366 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestSystemConfiguration(t *testing.T) { - t.Run("correct configuration is returned for system db collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - systemDbCollector := clickhouse.NewSystemDatabaseCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := systemDbCollector.Configuration() - require.Len(t, conf.Params, 3) - // check first param - require.IsType(t, config.StringListParam{}, conf.Params[0]) - includeTables, ok := conf.Params[0].(config.StringListParam) - require.True(t, ok) - require.False(t, includeTables.Required()) - require.Equal(t, includeTables.Name(), "include_tables") - require.Nil(t, includeTables.Values) - // check second param - require.IsType(t, config.StringListParam{}, conf.Params[1]) - excludeTables, ok := conf.Params[1].(config.StringListParam) - require.True(t, ok) - require.False(t, excludeTables.Required()) - require.Equal(t, "exclude_tables", excludeTables.Name()) - require.Equal(t, []string{"licenses", "distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper", "aggregate_function_combinators", "collations", "contributors", "data_type_families", "formats", "graphite_retentions", "numbers", "numbers_mt", "one", "parts_columns", "projection_parts", "projection_parts_columns", "table_engines", "time_zones", "zeros", "zeros_mt"}, excludeTables.Values) - // check third param - require.IsType(t, config.IntParam{}, conf.Params[2]) - rowLimit, ok := conf.Params[2].(config.IntParam) - require.True(t, ok) - require.False(t, rowLimit.Required()) - require.Equal(t, "row_limit", rowLimit.Name()) - require.Equal(t, int64(100000), rowLimit.Value) - }) -} - -func TestSystemDbCollect(t *testing.T) { - - diskFrame := test.NewFakeDataFrame("disks", []string{"name", "path", "free_space", "total_space", "keep_free_space", "type"}, - [][]interface{}{ - {"default", "/var/lib/clickhouse", 1729659346944, 1938213220352, "", "local"}, - }, - ) - clusterFrame := test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - {"events", 2, 1, 1, "dalem-local-clickhouse-blue-2", "192.168.144.4", 9000, 1, "default", "", 0, 0, 0}, - {"events", 3, 1, 1, "dalem-local-clickhouse-blue-3", "192.168.144.3", 9000, 1, "default", "", 0, 0, 0}, - }, - ) - userFrame := test.NewFakeDataFrame("users", []string{"name", "id", "storage", "auth_type", "auth_params", "host_ip", "host_names", "host_names_regexp", "host_names_like"}, - [][]interface{}{ - {"default", "94309d50-4f52-5250-31bd-74fecac179db,users.xml,plaintext_password", "sha256_password", []string{"::0"}, []string{}, []string{}, []string{}}, - }, - ) - - dbTables := map[string][]string{ - clickhouse.SystemDatabase: {"disks", "clusters", "users"}, - } - client := test.NewFakeClickhouseClient(dbTables) - - client.QueryResponses["SELECT * FROM system.disks LIMIT 100000"] = &diskFrame - client.QueryResponses["SELECT * FROM system.clusters LIMIT 100000"] = &clusterFrame - client.QueryResponses["SELECT * FROM system.users LIMIT 100000"] = &userFrame - systemDbCollector := clickhouse.NewSystemDatabaseCollector(&platform.ResourceManager{ - DbClient: client, - }) - - t.Run("test default system db collection", func(t *testing.T) { - diagSet, err := systemDbCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - checkFrame(t, diagSet.Frames["clusters"], clusterFrame.Rows) - // users frame - require.Equal(t, "users", diagSet.Frames["users"].Name()) - require.Equal(t, userFrame.ColumnNames, diagSet.Frames["users"].Columns()) - checkFrame(t, diagSet.Frames["users"], userFrame.Rows) - client.Reset() - }) - - t.Run("test when we pass an includes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks"}, - Param: config.NewParam("include_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 1) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - client.Reset() - }) - - // test excludes - t.Run("test when we pass an excludes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - Values: []string{"disks"}, - Param: config.NewParam("exclude_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 2) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - checkFrame(t, diagSet.Frames["clusters"], clusterFrame.Rows) - // users frame - require.Equal(t, "users", diagSet.Frames["users"].Name()) - require.Equal(t, userFrame.ColumnNames, diagSet.Frames["users"].Columns()) - checkFrame(t, diagSet.Frames["users"], userFrame.Rows) - client.Reset() - }) - - // test includes which isn't in the list - t.Run("test when we pass an invalid includes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks", "invalid"}, - Param: config.NewParam("include_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 1) - require.Equal(t, diagSet.Errors.Error(), "some tables specified in the include_tables are not in the "+ - "system database and will not be exported: [invalid]") - require.Len(t, diagSet.Frames, 1) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - client.Reset() - }) - - t.Run("test when we use a table with excluded fields", func(t *testing.T) { - excludeDefault := clickhouse.ExcludeColumns - client.QueryResponses["SELECT * EXCEPT(keep_free_space,type) FROM system.disks LIMIT 100000"] = &diskFrame - clickhouse.ExcludeColumns = map[string][]string{ - "disks": {"keep_free_space", "type"}, - } - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks"}, - Param: config.NewParam("include_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 1) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, []string{"name", "path", "free_space", "total_space"}, diagSet.Frames["disks"].Columns()) - eDiskFrame := test.NewFakeDataFrame("disks", []string{"name", "path", "free_space", "total_space"}, - [][]interface{}{ - {"default", "/var/lib/clickhouse", 1729659346944, 1938213220352}, - }, - ) - checkFrame(t, diagSet.Frames["disks"], eDiskFrame.Rows) - clickhouse.ExcludeColumns = excludeDefault - client.Reset() - }) - - t.Run("test with a low row limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 1, - Param: config.NewParam("row_limit", "Maximum number of rows to collect from any table. Negative values mean unlimited.", false), - }, - }, - } - client.QueryResponses["SELECT * FROM system.disks LIMIT 1"] = &diskFrame - client.QueryResponses["SELECT * FROM system.clusters LIMIT 1"] = &clusterFrame - client.QueryResponses["SELECT * FROM system.users LIMIT 1"] = &userFrame - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - lClusterFrame := test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - }) - checkFrame(t, diagSet.Frames["clusters"], lClusterFrame.Rows) - client.Reset() - }) - - t.Run("test with a negative low row limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: -23, - Param: config.NewParam("row_limit", "Maximum number of rows to collect from any table. Negative values mean unlimited.", false), - }, - }, - } - client.QueryResponses["SELECT * FROM system.clusters"] = &clusterFrame - client.QueryResponses["SELECT * FROM system.disks"] = &diskFrame - client.QueryResponses["SELECT * FROM system.users"] = &userFrame - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - checkFrame(t, diagSet.Frames["clusters"], clusterFrame.Rows) - // users frame - require.Equal(t, "users", diagSet.Frames["users"].Name()) - require.Equal(t, userFrame.ColumnNames, diagSet.Frames["users"].Columns()) - checkFrame(t, diagSet.Frames["users"], userFrame.Rows) - client.Reset() - }) - - t.Run("test that includes overrides excludes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks"}, - Param: config.NewParam("exclude_tables", "Excluded", false), - }, - config.StringListParam{ - // nil means include everything - Values: []string{"disks", "clusters", "users"}, - Param: config.NewParam("include_tables", "Included", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - client.Reset() - }) - - t.Run("test banned", func(t *testing.T) { - bannedDefault := clickhouse.BannedTables - clickhouse.BannedTables = []string{"disks"} - diagSet, err := systemDbCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 2) - require.Contains(t, diagSet.Frames, "users") - require.Contains(t, diagSet.Frames, "clusters") - clickhouse.BannedTables = bannedDefault - client.Reset() - }) - - t.Run("test banned unless included", func(t *testing.T) { - bannedDefault := clickhouse.BannedTables - clickhouse.BannedTables = []string{"disks"} - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks", "clusters", "users"}, - Param: config.NewParam("include_tables", "Included", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - require.Contains(t, diagSet.Frames, "disks") - require.Contains(t, diagSet.Frames, "users") - require.Contains(t, diagSet.Frames, "clusters") - clickhouse.BannedTables = bannedDefault - client.Reset() - }) - - t.Run("tables are ordered if configured", func(t *testing.T) { - defaultOrderBy := clickhouse.OrderBy - clickhouse.OrderBy = map[string]data.OrderBy{ - "clusters": { - Column: "shard_num", - Order: data.Desc, - }, - } - client.QueryResponses["SELECT * FROM system.clusters ORDER BY shard_num DESC LIMIT 100000"] = &clusterFrame - diagSet, err := systemDbCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - clickhouse.OrderBy = defaultOrderBy - oClusterFrame := test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 3, 1, 1, "dalem-local-clickhouse-blue-3", "192.168.144.3", 9000, 1, "default", "", 0, 0, 0}, - {"events", 2, 1, 1, "dalem-local-clickhouse-blue-2", "192.168.144.4", 9000, 1, "default", "", 0, 0, 0}, - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - }, - ) - checkFrame(t, diagSet.Frames["clusters"], oClusterFrame.Rows) - client.Reset() - }) - -} - -func checkFrame(t *testing.T, frame data.Frame, rows [][]interface{}) { - i := 0 - for { - values, ok, err := frame.Next() - require.Nil(t, err) - if !ok { - break - } - require.ElementsMatch(t, rows[i], values) - i += 1 - } - require.Equal(t, i, len(rows)) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go deleted file mode 100644 index 78aefeaa0c1..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go +++ /dev/null @@ -1,153 +0,0 @@ -package clickhouse - -import ( - "fmt" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/bmatcuk/doublestar/v4" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -// This collector collects the system zookeeper db - -type ZookeeperCollector struct { - resourceManager *platform.ResourceManager -} - -func NewZookeeperCollector(m *platform.ResourceManager) *ZookeeperCollector { - return &ZookeeperCollector{ - resourceManager: m, - } -} - -func (zkc *ZookeeperCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(zkc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - - pathPattern, err := config.ReadStringValue(conf, "path_pattern") - if err != nil { - return &data.DiagnosticBundle{}, err - } - defaultPattern, _ := zkc.Configuration().GetConfigParam("path_pattern") - if defaultPattern.(config.StringParam).Value != pathPattern { - log.Warn().Msgf("Using non default zookeeper glob pattern [%s] - this can potentially cause high query load", pathPattern) - } - maxDepth, err := config.ReadIntValue(conf, "max_depth") - if err != nil { - return &data.DiagnosticBundle{}, err - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - // we use doublestar for globs as it provides us with ** but also allows us to identify prefix or base paths - if !doublestar.ValidatePattern(pathPattern) { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "%s is not a valid pattern", pathPattern) - } - base, _ := doublestar.SplitPattern(pathPattern) - frames := make(map[string]data.Frame) - hFrame, frameErrors := zkc.collectSubFrames(base, pathPattern, rowLimit, 0, maxDepth) - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - frames["zookeeper_db"] = hFrame - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -// recursively iterates over the zookeeper sub tables to a max depth, applying the filter and max rows per table -func (zkc *ZookeeperCollector) collectSubFrames(path, pathPattern string, rowLimit, currentDepth, maxDepth int64) (data.HierarchicalFrame, []error) { - var frameErrors []error - var subFrames []data.HierarchicalFrame - - currentDepth += 1 - if currentDepth == maxDepth { - return data.HierarchicalFrame{}, frameErrors - } - match, err := doublestar.PathMatch(pathPattern, path) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Path match failed for pattern %s with path %s", pathPattern, path)) - return data.HierarchicalFrame{}, frameErrors - } - // we allow a single level to be examined or we never get going - if !match && currentDepth > 1 { - return data.HierarchicalFrame{}, frameErrors - } - frame, err := zkc.resourceManager.DbClient.ExecuteStatement(path, fmt.Sprintf("SELECT name FROM system.zookeeper WHERE path='%s' LIMIT %d", path, rowLimit)) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to read zookeeper table path for sub paths %s", path)) - return data.HierarchicalFrame{}, frameErrors - } - - // this isn't ideal, we add re-execute the query to our collection as this will be consumed by the output lazily - outputFrame, err := zkc.resourceManager.DbClient.ExecuteStatement(path, fmt.Sprintf("SELECT * FROM system.zookeeper WHERE path='%s' LIMIT %d", path, rowLimit)) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to read zookeeper table path %s", path)) - return data.HierarchicalFrame{}, frameErrors - } - frameComponents := strings.Split(path, "/") - frameId := frameComponents[len(frameComponents)-1] - - for { - values, ok, err := frame.Next() - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "unable to read frame %s", frame.Name())) - return data.NewHierarchicalFrame(frameId, outputFrame, subFrames), frameErrors - } - if !ok { - return data.NewHierarchicalFrame(frameId, outputFrame, subFrames), frameErrors - } - subName := fmt.Sprintf("%v", values[0]) - subPath := fmt.Sprintf("%s/%s", path, subName) - subFrame, errs := zkc.collectSubFrames(subPath, pathPattern, rowLimit, currentDepth, maxDepth) - if subFrame.Name() != "" { - subFrames = append(subFrames, subFrame) - } - frameErrors = append(frameErrors, errs...) - } -} - -func (zkc *ZookeeperCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "/clickhouse/{task_queue}/**", - Param: config.NewParam("path_pattern", "Glob pattern for zookeeper path matching. Change with caution.", false), - }, - config.IntParam{ - Value: 8, - Param: config.NewParam("max_depth", "Max depth for zookeeper navigation.", false), - }, - config.IntParam{ - Value: 10, - Param: config.NewParam("row_limit", "Maximum number of rows/sub nodes to collect/expand from any zookeeper leaf. Negative values mean unlimited.", false), - }, - }, - } -} - -func (zkc *ZookeeperCollector) IsDefault() bool { - return false -} - -func (zkc *ZookeeperCollector) Description() string { - return "Collects Zookeeper information available within ClickHouse." -} - -// here we register the collector for use -func init() { - collectors.Register("zookeeper_db", func() (collectors.Collector, error) { - return &ZookeeperCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go deleted file mode 100644 index 3e56f6200f0..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestZookeeperConfiguration(t *testing.T) { - t.Run("correct configuration is returned for system zookeeper collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - zkCollector := clickhouse.NewZookeeperCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := zkCollector.Configuration() - require.Len(t, conf.Params, 3) - // check first param - require.IsType(t, config.StringParam{}, conf.Params[0]) - pathPattern, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, pathPattern.Required()) - require.Equal(t, pathPattern.Name(), "path_pattern") - require.Equal(t, "/clickhouse/{task_queue}/**", pathPattern.Value) - // check second param - require.IsType(t, config.IntParam{}, conf.Params[1]) - maxDepth, ok := conf.Params[1].(config.IntParam) - require.True(t, ok) - require.False(t, maxDepth.Required()) - require.Equal(t, "max_depth", maxDepth.Name()) - require.Equal(t, int64(8), maxDepth.Value) - // check third param - require.IsType(t, config.IntParam{}, conf.Params[2]) - rowLimit, ok := conf.Params[2].(config.IntParam) - require.True(t, ok) - require.False(t, rowLimit.Required()) - require.Equal(t, "row_limit", rowLimit.Name()) - require.Equal(t, int64(10), rowLimit.Value) - }) -} - -func TestZookeeperCollect(t *testing.T) { - level1 := test.NewFakeDataFrame("level_1", []string{"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - [][]interface{}{ - {"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - {"task_queue", "", "4", "4", "2022-02-22 13:30:15", "2022-02-22 13:30:15", "0", "1", "0", "0", "0", "1", "5", "/clickhouse"}, - {"copytasks", "", "525608", "525608", "2022-03-09 13:47:39", "2022-03-09 13:47:39", "0", "7", "0", "0", "0", "7", "526100", "/clickhouse"}, - }, - ) - level2 := test.NewFakeDataFrame("level_2", []string{"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - [][]interface{}{ - {"ddl", "", "5", "5", "2022-02-22 13:30:15", "2022-02-22 13:30:15", "0", "0", "0", "0", "0", "0", "5", "/clickhouse/task_queue"}, - }, - ) - level3 := test.NewFakeDataFrame("level_2", []string{"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - [][]interface{}{}, - ) - dbTables := map[string][]string{ - clickhouse.SystemDatabase: {"zookeeper"}, - } - client := test.NewFakeClickhouseClient(dbTables) - - client.QueryResponses["SELECT name FROM system.zookeeper WHERE path='/clickhouse' LIMIT 10"] = &level1 - // can't reuse the frame as the first frame will be iterated as part of the recursive zookeeper search performed by the collector - cLevel1 := test.NewFakeDataFrame("level_1", level1.Columns(), level1.Rows) - client.QueryResponses["SELECT * FROM system.zookeeper WHERE path='/clickhouse' LIMIT 10"] = &cLevel1 - client.QueryResponses["SELECT name FROM system.zookeeper WHERE path='/clickhouse/task_queue' LIMIT 10"] = &level2 - cLevel2 := test.NewFakeDataFrame("level_2", level2.Columns(), level2.Rows) - client.QueryResponses["SELECT * FROM system.zookeeper WHERE path='/clickhouse/task_queue' LIMIT 10"] = &cLevel2 - client.QueryResponses["SELECT name FROM system.zookeeper WHERE path='/clickhouse/task_queue/ddl' LIMIT 10"] = &level3 - cLevel3 := test.NewFakeDataFrame("level_3", level3.Columns(), level3.Rows) - client.QueryResponses["SELECT * FROM system.zookeeper WHERE path='/clickhouse/task_queue/ddl' LIMIT 10"] = &cLevel3 - - zKCollector := clickhouse.NewZookeeperCollector(&platform.ResourceManager{ - DbClient: client, - }) - - t.Run("test default zookeeper collection", func(t *testing.T) { - diagSet, err := zKCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 1) - require.Contains(t, diagSet.Frames, "zookeeper_db") - require.Equal(t, "clickhouse", diagSet.Frames["zookeeper_db"].Name()) - require.IsType(t, data.HierarchicalFrame{}, diagSet.Frames["zookeeper_db"]) - checkFrame(t, diagSet.Frames["zookeeper_db"], level1.Rows) - require.Equal(t, level1.Columns(), diagSet.Frames["zookeeper_db"].Columns()) - hierarchicalFrame := diagSet.Frames["zookeeper_db"].(data.HierarchicalFrame) - require.Len(t, hierarchicalFrame.SubFrames, 1) - checkFrame(t, hierarchicalFrame.SubFrames[0], cLevel2.Rows) - require.Equal(t, cLevel2.Columns(), hierarchicalFrame.SubFrames[0].Columns()) - hierarchicalFrame = hierarchicalFrame.SubFrames[0] - require.Len(t, hierarchicalFrame.SubFrames, 1) - checkFrame(t, hierarchicalFrame.SubFrames[0], cLevel3.Rows) - require.Equal(t, cLevel3.Columns(), hierarchicalFrame.SubFrames[0].Columns()) - }) -} diff --git a/programs/diagnostics/internal/collectors/registry.go b/programs/diagnostics/internal/collectors/registry.go deleted file mode 100644 index 5611f947466..00000000000 --- a/programs/diagnostics/internal/collectors/registry.go +++ /dev/null @@ -1,75 +0,0 @@ -package collectors - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -type Collector interface { - Collect(config config.Configuration) (*data.DiagnosticBundle, error) - Configuration() config.Configuration - IsDefault() bool - Description() string -} - -// Register can be called from init() on a collector in this package -// It will automatically be added to the Collectors map to be called externally -func Register(name string, collector CollectorFactory) { - if name == "diag_trace" { - // we use this to record errors and warnings - log.Fatal().Msgf("diag_trace is a reserved collector name") - } - // names must be unique - if _, ok := Collectors[name]; ok { - log.Fatal().Msgf("More than 1 collector is trying to register under the name %s. Names must be unique.", name) - } - Collectors[name] = collector -} - -// CollectorFactory lets us use a closure to get instances of the collector struct -type CollectorFactory func() (Collector, error) - -var Collectors = map[string]CollectorFactory{} - -func GetCollectorNames(defaultOnly bool) []string { - // can't pre-allocate as not all maybe default - var collectors []string - for collectorName := range Collectors { - collector, err := GetCollectorByName(collectorName) - if err != nil { - log.Fatal().Err(err) - } - if !defaultOnly || (defaultOnly && collector.IsDefault()) { - collectors = append(collectors, collectorName) - } - } - return collectors -} - -func GetCollectorByName(name string) (Collector, error) { - if collectorFactory, ok := Collectors[name]; ok { - //do something here - collector, err := collectorFactory() - if err != nil { - return nil, errors.Wrapf(err, "collector %s could not be initialized", name) - } - return collector, nil - } - return nil, fmt.Errorf("%s is not a valid collector name", name) -} - -func BuildConfigurationOptions() (map[string]config.Configuration, error) { - configurations := make(map[string]config.Configuration) - for name, collectorFactory := range Collectors { - collector, err := collectorFactory() - if err != nil { - return nil, errors.Wrapf(err, "collector %s could not be initialized", name) - } - configurations[name] = collector.Configuration() - } - return configurations, nil -} diff --git a/programs/diagnostics/internal/collectors/registry_test.go b/programs/diagnostics/internal/collectors/registry_test.go deleted file mode 100644 index eccc5f2265d..00000000000 --- a/programs/diagnostics/internal/collectors/registry_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package collectors_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/stretchr/testify/require" -) - -func TestGetCollectorNames(t *testing.T) { - t.Run("can get all collector names", func(t *testing.T) { - collectorNames := collectors.GetCollectorNames(false) - require.ElementsMatch(t, []string{"system_db", "config", "summary", "system", "logs", "db_logs", "file", "command", "zookeeper_db"}, collectorNames) - }) - - t.Run("can get default collector names", func(t *testing.T) { - collectorNames := collectors.GetCollectorNames(true) - require.ElementsMatch(t, []string{"system_db", "config", "summary", "system", "logs", "db_logs"}, collectorNames) - }) -} - -func TestGetCollectorByName(t *testing.T) { - - t.Run("can get collector by name", func(t *testing.T) { - collector, err := collectors.GetCollectorByName("system_db") - require.Nil(t, err) - require.Equal(t, clickhouse.NewSystemDatabaseCollector(platform.GetResourceManager()), collector) - }) - - t.Run("fails on non existing collector", func(t *testing.T) { - collector, err := collectors.GetCollectorByName("random") - require.NotNil(t, err) - require.Equal(t, "random is not a valid collector name", err.Error()) - require.Nil(t, collector) - }) -} - -func TestBuildConfigurationOptions(t *testing.T) { - - t.Run("can get all collector configurations", func(t *testing.T) { - configs, err := collectors.BuildConfigurationOptions() - require.Nil(t, err) - require.Len(t, configs, 9) - require.Contains(t, configs, "system_db") - require.Contains(t, configs, "config") - require.Contains(t, configs, "summary") - require.Contains(t, configs, "system") - require.Contains(t, configs, "logs") - require.Contains(t, configs, "db_logs") - require.Contains(t, configs, "file") - require.Contains(t, configs, "command") - require.Contains(t, configs, "zookeeper_db") - }) -} diff --git a/programs/diagnostics/internal/collectors/system/command.go b/programs/diagnostics/internal/collectors/system/command.go deleted file mode 100644 index ba4dd1e996c..00000000000 --- a/programs/diagnostics/internal/collectors/system/command.go +++ /dev/null @@ -1,90 +0,0 @@ -package system - -import ( - "bytes" - "os/exec" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/google/shlex" - "github.com/pkg/errors" -) - -// This collector runs a user specified command and collects it to a file - -type CommandCollector struct { - resourceManager *platform.ResourceManager -} - -func NewCommandCollector(m *platform.ResourceManager) *CommandCollector { - return &CommandCollector{ - resourceManager: m, - } -} - -func (c *CommandCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(c.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - command, err := config.ReadStringValue(conf, "command") - if err != nil { - return &data.DiagnosticBundle{}, err - } - var frameErrors []error - // shlex to split the commands and args - cmdArgs, err := shlex.Split(command) - if err != nil || len(cmdArgs) == 0 { - return &data.DiagnosticBundle{}, errors.Wrap(err, "Unable to parse command") - } - cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err = cmd.Run() - var sError string - if err != nil { - frameErrors = append(frameErrors, errors.Wrap(err, "Unable to execute command")) - sError = err.Error() - } - memoryFrame := data.NewMemoryFrame("output", []string{"command", "stdout", "stderr", "error"}, [][]interface{}{ - {command, stdout.String(), stderr.String(), sError}, - }) - return &data.DiagnosticBundle{ - Errors: data.FrameErrors{Errors: frameErrors}, - Frames: map[string]data.Frame{ - "output": memoryFrame, - }, - }, nil -} - -func (c *CommandCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - } -} - -func (c *CommandCollector) IsDefault() bool { - return false -} - -func (c *CommandCollector) Description() string { - return "Allows collection of the output from a user specified command" -} - -// here we register the collector for use -func init() { - collectors.Register("command", func() (collectors.Collector, error) { - return &CommandCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/system/command_test.go b/programs/diagnostics/internal/collectors/system/command_test.go deleted file mode 100644 index 7de00cdabf4..00000000000 --- a/programs/diagnostics/internal/collectors/system/command_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package system_test - -import ( - "fmt" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestCommandConfiguration(t *testing.T) { - t.Run("correct configuration is returned for file collector", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - conf := commandCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.StringParam{}, conf.Params[0]) - command, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.True(t, command.Required()) - require.Equal(t, command.Name(), "command") - require.Equal(t, "", command.Value) - }) -} - -func TestCommandCollect(t *testing.T) { - t.Run("test simple command with args", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - bundle, err := commandCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "ls -l ../../../testdata", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - }) - require.Nil(t, err) - require.Nil(t, bundle.Errors.Errors) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "output") - require.Equal(t, bundle.Frames["output"].Columns(), []string{"command", "stdout", "stderr", "error"}) - memFrame := bundle.Frames["output"].(data.MemoryFrame) - values, ok, err := memFrame.Next() - require.True(t, ok) - require.Nil(t, err) - fmt.Println(values) - require.Len(t, values, 4) - require.Equal(t, "ls -l ../../../testdata", values[0]) - require.Contains(t, values[1], "configs") - require.Contains(t, values[1], "docker") - require.Contains(t, values[1], "log") - require.Equal(t, "", values[2]) - require.Equal(t, "", values[3]) - values, ok, err = memFrame.Next() - require.False(t, ok) - require.Nil(t, err) - require.Nil(t, values) - }) - - t.Run("test empty command", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - bundle, err := commandCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - }) - require.Equal(t, "parameter command is invalid - command cannot be empty", err.Error()) - require.Equal(t, &data.DiagnosticBundle{}, bundle) - }) - - t.Run("test invalid command", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - bundle, err := commandCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "ls --invalid ../../../testdata", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - }) - // commands may error with output - we still capture on stderr - require.Nil(t, err) - require.Len(t, bundle.Errors.Errors, 1) - require.Equal(t, "Unable to execute command: exit status 2", bundle.Errors.Errors[0].Error()) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "output") - require.Equal(t, bundle.Frames["output"].Columns(), []string{"command", "stdout", "stderr", "error"}) - memFrame := bundle.Frames["output"].(data.MemoryFrame) - values, ok, err := memFrame.Next() - require.True(t, ok) - require.Nil(t, err) - require.Len(t, values, 4) - require.Equal(t, "ls --invalid ../../../testdata", values[0]) - require.Equal(t, "", values[1]) - // exact values here may vary on platform - require.NotEmpty(t, values[2]) - require.NotEmpty(t, values[3]) - }) -} diff --git a/programs/diagnostics/internal/collectors/system/file.go b/programs/diagnostics/internal/collectors/system/file.go deleted file mode 100644 index cda91636c52..00000000000 --- a/programs/diagnostics/internal/collectors/system/file.go +++ /dev/null @@ -1,100 +0,0 @@ -package system - -import ( - "os" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/yargevad/filepathx" -) - -// This collector collects arbitrary user files - -type FileCollector struct { - resourceManager *platform.ResourceManager -} - -func NewFileCollector(m *platform.ResourceManager) *FileCollector { - return &FileCollector{ - resourceManager: m, - } -} - -func (f *FileCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(f.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - filePattern, err := config.ReadStringValue(conf, "file_pattern") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - var frameErrors []error - // this util package supports recursive file matching e.g. /**/* - matches, err := filepathx.Glob(filePattern) - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "Invalid file_pattern \"%s\"", filePattern) - } - - if len(matches) == 0 { - frameErrors = append(frameErrors, errors.New("0 files match glob pattern")) - return &data.DiagnosticBundle{ - Errors: data.FrameErrors{Errors: frameErrors}, - }, nil - } - - var filePaths []string - for _, match := range matches { - fi, err := os.Stat(match) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to read file %s", match)) - } - if !fi.IsDir() { - log.Debug().Msgf("Collecting file %s", match) - filePaths = append(filePaths, match) - } - } - - frame := data.NewFileFrame("collection", filePaths) - - return &data.DiagnosticBundle{ - Errors: data.FrameErrors{Errors: frameErrors}, - Frames: map[string]data.Frame{ - "collection": frame, - }, - }, nil -} - -func (f *FileCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - } -} - -func (f *FileCollector) IsDefault() bool { - return false -} - -func (f *FileCollector) Description() string { - return "Allows collection of user specified files" -} - -// here we register the collector for use -func init() { - collectors.Register("file", func() (collectors.Collector, error) { - return &FileCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/system/file_test.go b/programs/diagnostics/internal/collectors/system/file_test.go deleted file mode 100644 index 5b1d5b3a92f..00000000000 --- a/programs/diagnostics/internal/collectors/system/file_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package system_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestFileConfiguration(t *testing.T) { - t.Run("correct configuration is returned for file collector", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - conf := fileCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.StringParam{}, conf.Params[0]) - filePattern, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.True(t, filePattern.Required()) - require.Equal(t, filePattern.Name(), "file_pattern") - require.Equal(t, "", filePattern.Value) - }) -} - -func TestFileCollect(t *testing.T) { - - t.Run("test filter patterns work", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - bundle, err := fileCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "../../../testdata/**/*.xml", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - }) - require.Nil(t, err) - require.Nil(t, bundle.Errors.Errors) - checkFileBundle(t, bundle, - []string{"../../../testdata/configs/include/xml/server-include.xml", - "../../../testdata/configs/include/xml/user-include.xml", - "../../../testdata/configs/xml/config.xml", - "../../../testdata/configs/xml/users.xml", - "../../../testdata/configs/xml/users.d/default-password.xml", - "../../../testdata/configs/yandex_xml/config.xml", - "../../../testdata/docker/admin.xml", - "../../../testdata/docker/custom.xml"}) - }) - - t.Run("invalid file patterns are detected", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - bundle, err := fileCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - }) - require.NotNil(t, err) - require.Equal(t, "parameter file_pattern is invalid - file_pattern cannot be empty", err.Error()) - require.Equal(t, &data.DiagnosticBundle{}, bundle) - }) - - t.Run("check empty matches are reported", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - bundle, err := fileCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "../../../testdata/**/*.random", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - }) - require.Nil(t, err) - require.Nil(t, bundle.Frames) - require.Len(t, bundle.Errors.Errors, 1) - require.Equal(t, "0 files match glob pattern", bundle.Errors.Errors[0].Error()) - }) - -} - -func checkFileBundle(t *testing.T, bundle *data.DiagnosticBundle, expectedFiles []string) { - require.NotNil(t, bundle) - require.Nil(t, bundle.Errors.Errors) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "collection") - dirFrame, ok := bundle.Frames["collection"].(data.FileFrame) - require.True(t, ok) - require.Equal(t, []string{"files"}, dirFrame.Columns()) - i := 0 - for { - values, ok, err := dirFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - file, ok := values[0].(data.SimpleFile) - require.True(t, ok) - require.Contains(t, expectedFiles, file.FilePath()) - i += 1 - } - require.Equal(t, len(expectedFiles), i) -} diff --git a/programs/diagnostics/internal/collectors/system/system.go b/programs/diagnostics/internal/collectors/system/system.go deleted file mode 100644 index 69d16f36b8b..00000000000 --- a/programs/diagnostics/internal/collectors/system/system.go +++ /dev/null @@ -1,235 +0,0 @@ -package system - -import ( - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/elastic/gosigar" - "github.com/jaypipes/ghw" - "github.com/matishsiao/goInfo" - "github.com/pkg/errors" -) - -// This collector collects the system overview - -type SystemCollector struct { - resourceManager *platform.ResourceManager -} - -func NewSystemCollector(m *platform.ResourceManager) *SystemCollector { - return &SystemCollector{ - resourceManager: m, - } -} - -func (sc *SystemCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - _, err := conf.ValidateConfig(sc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - frames := make(map[string]data.Frame) - var frameErrors []error - - frameErrors = addStatsToFrame(frames, frameErrors, "disks", getDisk) - frameErrors = addStatsToFrame(frames, frameErrors, "disk_usage", getDiskUsage) - - frameErrors = addStatsToFrame(frames, frameErrors, "memory", getMemory) - frameErrors = addStatsToFrame(frames, frameErrors, "memory_usage", getMemoryUsage) - - frameErrors = addStatsToFrame(frames, frameErrors, "cpu", getCPU) - //frameErrors = addStatsToFrame(frames, frameErrors, "cpu_usage", getCPUUsage) - - frameErrors = addStatsToFrame(frames, frameErrors, "processes", getProcessList) - - frameErrors = addStatsToFrame(frames, frameErrors, "os", getHostDetails) - - return &data.DiagnosticBundle{ - Frames: frames, - Errors: data.FrameErrors{ - Errors: frameErrors, - }, - }, err -} - -func addStatsToFrame(frames map[string]data.Frame, errors []error, name string, statFunc func() (data.MemoryFrame, error)) []error { - frame, err := statFunc() - if err != nil { - errors = append(errors, err) - } - frames[name] = frame - return errors -} - -func (sc *SystemCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{}, - } -} - -func (sc *SystemCollector) IsDefault() bool { - return true -} - -func getDisk() (data.MemoryFrame, error) { - block, err := ghw.Block() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list block storage") - } - var rows [][]interface{} - columns := []string{"name", "size", "physicalBlockSize", "driveType", "controller", "vendor", "model", "partitionName", "partitionSize", "mountPoint", "readOnly"} - for _, disk := range block.Disks { - for _, part := range disk.Partitions { - rows = append(rows, []interface{}{disk.Name, disk.SizeBytes, disk.PhysicalBlockSizeBytes, disk.DriveType, disk.StorageController, disk.Vendor, disk.Model, part.Name, part.SizeBytes, part.MountPoint, part.IsReadOnly}) - } - } - return data.NewMemoryFrame("disk_usage", columns, rows), nil -} - -func getDiskUsage() (data.MemoryFrame, error) { - fsList := gosigar.FileSystemList{} - err := fsList.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list filesystems for usage") - } - rows := make([][]interface{}, len(fsList.List)) - columns := []string{"filesystem", "size", "used", "avail", "use%", "mounted on"} - for i, fs := range fsList.List { - dirName := fs.DirName - usage := gosigar.FileSystemUsage{} - err = usage.Get(dirName) - if err == nil { - rows[i] = []interface{}{fs.DevName, usage.Total, usage.Used, usage.Avail, usage.UsePercent(), dirName} - } else { - // we try to output something - rows[i] = []interface{}{fs.DevName, 0, 0, 0, 0, dirName} - } - } - return data.NewMemoryFrame("disk_usage", columns, rows), nil -} - -func getMemory() (data.MemoryFrame, error) { - memory, err := ghw.Memory() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to read memory") - } - columns := []string{"totalPhysical", "totalUsable", "supportedPageSizes"} - rows := make([][]interface{}, 1) - rows[0] = []interface{}{memory.TotalPhysicalBytes, memory.TotalUsableBytes, memory.SupportedPageSizes} - return data.NewMemoryFrame("memory", columns, rows), nil -} - -func getMemoryUsage() (data.MemoryFrame, error) { - mem := gosigar.Mem{} - swap := gosigar.Swap{} - - err := mem.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to read memory usage") - } - - err = swap.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to read swap") - } - - columns := []string{"type", "total", "used", "free"} - rows := make([][]interface{}, 3) - - rows[0] = []interface{}{"mem", mem.Total, mem.Used, mem.Free} - rows[1] = []interface{}{"buffers/cache", 0, mem.ActualUsed, mem.ActualFree} - rows[2] = []interface{}{"swap", swap.Total, swap.Used, swap.Free} - return data.NewMemoryFrame("memory_usage", columns, rows), nil - -} - -func getCPU() (data.MemoryFrame, error) { - cpu, err := ghw.CPU() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list cpus") - } - columns := []string{"processor", "vendor", "model", "core", "numThreads", "logical", "capabilities"} - var rows [][]interface{} - for _, proc := range cpu.Processors { - for _, core := range proc.Cores { - rows = append(rows, []interface{}{proc.ID, proc.Vendor, proc.Model, core.ID, core.NumThreads, core.LogicalProcessors, strings.Join(proc.Capabilities, " ")}) - } - } - return data.NewMemoryFrame("cpu", columns, rows), nil -} - -// this gets cpu usage vs a listing of arch etc - see getCPU(). This needs successive values as its ticks - not currently used -// see https://github.com/elastic/beats/blob/master/metricbeat/internal/metrics/cpu/metrics.go#L131 for inspiration -//nolint -func getCPUUsage() (data.MemoryFrame, error) { - cpuList := gosigar.CpuList{} - err := cpuList.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list cpus for usage") - } - columns := []string{"sys", "nice", "stolen", "irq", "idle", "softIrq", "user", "wait", "total"} - rows := make([][]interface{}, len(cpuList.List), len(cpuList.List)) - for i, cpu := range cpuList.List { - rows[i] = []interface{}{cpu.Sys, cpu.Nice, cpu.Stolen, cpu.Irq, cpu.Idle, cpu.SoftIrq, cpu.User, cpu.Wait, cpu.Total()} - } - return data.NewMemoryFrame("cpu_usage", columns, rows), nil -} - -func getProcessList() (data.MemoryFrame, error) { - pidList := gosigar.ProcList{} - err := pidList.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list processes") - } - columns := []string{"pid", "ppid", "stime", "time", "rss", "size", "faults", "minorFaults", "majorFaults", "user", "state", "priority", "nice", "command"} - rows := make([][]interface{}, len(pidList.List)) - for i, pid := range pidList.List { - state := gosigar.ProcState{} - mem := gosigar.ProcMem{} - time := gosigar.ProcTime{} - args := gosigar.ProcArgs{} - if err := state.Get(pid); err != nil { - continue - } - if err := mem.Get(pid); err != nil { - continue - } - if err := time.Get(pid); err != nil { - continue - } - if err := args.Get(pid); err != nil { - continue - } - rows[i] = []interface{}{pid, state.Ppid, time.FormatStartTime(), time.FormatTotal(), mem.Resident, mem.Size, - mem.PageFaults, mem.MinorFaults, mem.MajorFaults, state.Username, state.State, state.Priority, state.Nice, - strings.Join(args.List, " ")} - } - return data.NewMemoryFrame("process_list", columns, rows), nil -} - -func getHostDetails() (data.MemoryFrame, error) { - gi, err := goInfo.GetInfo() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to get host summary") - } - columns := []string{"hostname", "os", "goOs", "cpus", "core", "kernel", "platform"} - rows := [][]interface{}{ - {gi.Hostname, gi.OS, gi.GoOS, gi.CPUs, gi.Core, gi.Kernel, gi.Platform}, - } - return data.NewMemoryFrame("os", columns, rows), nil -} - -func (sc *SystemCollector) Description() string { - return "Collects summary OS and hardware statistics for the host" -} - -// here we register the collector for use -func init() { - collectors.Register("system", func() (collectors.Collector, error) { - return &SystemCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/system/system_test.go b/programs/diagnostics/internal/collectors/system/system_test.go deleted file mode 100644 index fb1e16bd1ed..00000000000 --- a/programs/diagnostics/internal/collectors/system/system_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package system_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestSystemConfiguration(t *testing.T) { - t.Run("correct configuration is returned for system collector", func(t *testing.T) { - systemCollector := system.NewSystemCollector(&platform.ResourceManager{}) - conf := systemCollector.Configuration() - require.Len(t, conf.Params, 0) - require.Equal(t, []config.ConfigParam{}, conf.Params) - }) -} - -func TestSystemCollect(t *testing.T) { - t.Run("test default system collection", func(t *testing.T) { - systemCollector := system.NewSystemCollector(&platform.ResourceManager{}) - diagSet, err := systemCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 7) - require.Contains(t, diagSet.Frames, "disks") - require.Contains(t, diagSet.Frames, "disk_usage") - require.Contains(t, diagSet.Frames, "memory") - require.Contains(t, diagSet.Frames, "memory_usage") - require.Contains(t, diagSet.Frames, "cpu") - require.Contains(t, diagSet.Frames, "processes") - require.Contains(t, diagSet.Frames, "os") - // responses here will vary depending on platform - mocking seems excessive so we test we have some data - // disks - require.Equal(t, []string{"name", "size", "physicalBlockSize", "driveType", "controller", "vendor", "model", "partitionName", "partitionSize", "mountPoint", "readOnly"}, diagSet.Frames["disks"].Columns()) - diskFrames, err := countFrameRows(diagSet, "disks") - require.Greater(t, diskFrames, 0) - require.Nil(t, err) - // disk usage - require.Equal(t, []string{"filesystem", "size", "used", "avail", "use%", "mounted on"}, diagSet.Frames["disk_usage"].Columns()) - diskUsageFrames, err := countFrameRows(diagSet, "disk_usage") - require.Greater(t, diskUsageFrames, 0) - require.Nil(t, err) - // memory - require.Equal(t, []string{"totalPhysical", "totalUsable", "supportedPageSizes"}, diagSet.Frames["memory"].Columns()) - memoryFrames, err := countFrameRows(diagSet, "memory") - require.Greater(t, memoryFrames, 0) - require.Nil(t, err) - // memory_usage - require.Equal(t, []string{"type", "total", "used", "free"}, diagSet.Frames["memory_usage"].Columns()) - memoryUsageFrames, err := countFrameRows(diagSet, "memory_usage") - require.Greater(t, memoryUsageFrames, 0) - require.Nil(t, err) - // cpu - require.Equal(t, []string{"processor", "vendor", "model", "core", "numThreads", "logical", "capabilities"}, diagSet.Frames["cpu"].Columns()) - cpuFrames, err := countFrameRows(diagSet, "cpu") - require.Greater(t, cpuFrames, 0) - require.Nil(t, err) - // processes - require.Equal(t, []string{"pid", "ppid", "stime", "time", "rss", "size", "faults", "minorFaults", "majorFaults", "user", "state", "priority", "nice", "command"}, diagSet.Frames["processes"].Columns()) - processesFrames, err := countFrameRows(diagSet, "processes") - require.Greater(t, processesFrames, 0) - require.Nil(t, err) - // os - require.Equal(t, []string{"hostname", "os", "goOs", "cpus", "core", "kernel", "platform"}, diagSet.Frames["os"].Columns()) - osFrames, err := countFrameRows(diagSet, "os") - require.Greater(t, osFrames, 0) - require.Nil(t, err) - }) -} - -func countFrameRows(diagSet *data.DiagnosticBundle, frameName string) (int, error) { - frame := diagSet.Frames[frameName] - i := 0 - for { - _, ok, err := frame.Next() - if !ok { - return i, err - } - if err != nil { - return i, err - } - i++ - } -} diff --git a/programs/diagnostics/internal/outputs/file/simple.go b/programs/diagnostics/internal/outputs/file/simple.go deleted file mode 100644 index 63847b3addd..00000000000 --- a/programs/diagnostics/internal/outputs/file/simple.go +++ /dev/null @@ -1,344 +0,0 @@ -package file - -import ( - "context" - "encoding/csv" - "fmt" - "os" - "path" - "path/filepath" - "strconv" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/mholt/archiver/v4" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -const OutputName = "simple" - -type SubFolderGenerator func() string - -type SimpleOutput struct { - // mainly used for testing to make sub folder deterministic - which it won't be by default as it uses a timestamp - FolderGenerator SubFolderGenerator -} - -func (o SimpleOutput) Write(id string, bundles map[string]*data.DiagnosticBundle, conf config.Configuration) (data.FrameErrors, error) { - conf, err := conf.ValidateConfig(o.Configuration()) - if err != nil { - return data.FrameErrors{}, err - } - directory, err := config.ReadStringValue(conf, "directory") - if err != nil { - return data.FrameErrors{}, err - } - directory, err = getWorkingDirectory(directory) - if err != nil { - return data.FrameErrors{}, err - } - subFolder := strconv.FormatInt(utils.MakeTimestamp(), 10) - if o.FolderGenerator != nil { - subFolder = o.FolderGenerator() - } - skipArchive, err := config.ReadBoolValue(conf, "skip_archive") - if err != nil { - return data.FrameErrors{}, err - } - - outputDir := filepath.Join(directory, id, subFolder) - log.Info().Msgf("creating bundle in %s", outputDir) - if err := os.MkdirAll(outputDir, os.ModePerm); err != nil { - return data.FrameErrors{}, err - } - frameErrors := data.FrameErrors{} - var filePaths []string - for name := range bundles { - bundlePaths, frameError := writeDiagnosticBundle(name, bundles[name], outputDir) - filePaths = append(filePaths, bundlePaths...) - frameErrors.Errors = append(frameErrors.Errors, frameError.Errors...) - } - log.Info().Msg("bundle created") - if !skipArchive { - archiveFilename := filepath.Join(directory, id, fmt.Sprintf("%s.tar.gz", subFolder)) - log.Info().Msgf("compressing bundle to %s", archiveFilename) - // produce a map containing the input paths to the archive paths - we preserve the output directory and hierarchy - archiveMap := createArchiveMap(filePaths, directory) - if err := createArchive(archiveFilename, archiveMap); err != nil { - return frameErrors, err - } - // we delete the original directory leaving just the archive behind - if err := os.RemoveAll(outputDir); err != nil { - return frameErrors, err - } - log.Info().Msgf("archive ready at: %s ", archiveFilename) - } - return frameErrors, nil -} - -func writeDiagnosticBundle(name string, diag *data.DiagnosticBundle, baseDir string) ([]string, data.FrameErrors) { - diagDir := filepath.Join(baseDir, name) - if err := os.MkdirAll(diagDir, os.ModePerm); err != nil { - return nil, data.FrameErrors{Errors: []error{ - errors.Wrapf(err, "unable to create directory for %s", name), - }} - } - frameErrors := data.FrameErrors{} - var filePaths []string - for frameId, frame := range diag.Frames { - fFilePath, errs := writeFrame(frameId, frame, diagDir) - filePaths = append(filePaths, fFilePath...) - if len(errs) > 0 { - // it would be nice if we could wrap this list of errors into something formal but this logs well - frameErrors.Errors = append(frameErrors.Errors, fmt.Errorf("unable to write frame %s for %s", frameId, name)) - frameErrors.Errors = append(frameErrors.Errors, errs...) - } - } - return filePaths, frameErrors -} - -func writeFrame(frameId string, frame data.Frame, baseDir string) ([]string, []error) { - switch f := frame.(type) { - case data.DatabaseFrame: - return writeDatabaseFrame(frameId, f, baseDir) - case data.ConfigFileFrame: - return writeConfigFrame(frameId, f, baseDir) - case data.DirectoryFileFrame: - return processDirectoryFileFrame(frameId, f, baseDir) - case data.FileFrame: - return processFileFrame(frameId, f, baseDir) - case data.HierarchicalFrame: - return writeHierarchicalFrame(frameId, f, baseDir) - default: - // for now our data frame writer supports all frames - return writeDatabaseFrame(frameId, frame, baseDir) - } -} - -func writeHierarchicalFrame(frameId string, frame data.HierarchicalFrame, baseDir string) ([]string, []error) { - filePaths, errs := writeFrame(frameId, frame.DataFrame, baseDir) - for _, subFrame := range frame.SubFrames { - subDir := filepath.Join(baseDir, subFrame.Name()) - if err := os.MkdirAll(subDir, os.ModePerm); err != nil { - errs = append(errs, err) - continue - } - subPaths, subErrs := writeFrame(subFrame.Name(), subFrame, subDir) - filePaths = append(filePaths, subPaths...) - errs = append(errs, subErrs...) - } - return filePaths, errs -} - -func writeDatabaseFrame(frameId string, frame data.Frame, baseDir string) ([]string, []error) { - frameFilePath := filepath.Join(baseDir, fmt.Sprintf("%s.csv", frameId)) - var errs []error - f, err := os.Create(frameFilePath) - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - defer f.Close() - w := csv.NewWriter(f) - defer w.Flush() - if err := w.Write(frame.Columns()); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to write columns for frame %s", frameId)) - return []string{}, errs - } - // we don't collect an error for every line here like configs and logs - could mean a lot of unnecessary noise - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to read frame %s", frameId)) - return []string{}, errs - } - if !ok { - return []string{frameFilePath}, errs - } - sValues := make([]string, len(values)) - for i, value := range values { - sValues[i] = fmt.Sprintf("%v", value) - } - if err := w.Write(sValues); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to write row for frame %s", frameId)) - return []string{}, errs - } - } -} - -func writeConfigFrame(frameId string, frame data.ConfigFileFrame, baseDir string) ([]string, []error) { - var errs []error - frameDirectory := filepath.Join(baseDir, frameId) - if err := os.MkdirAll(frameDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - // this holds our files included - includesDirectory := filepath.Join(frameDirectory, "includes") - if err := os.MkdirAll(includesDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create includes directory for frame %s", frameId)) - return []string{}, errs - } - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, err) - return []string{frameDirectory}, errs - } - if !ok { - return []string{frameDirectory}, errs - } - configFile := values[0].(data.ConfigFile) - if !configFile.IsIncluded() { - relPath := strings.TrimPrefix(configFile.FilePath(), frame.Directory) - destPath := path.Join(frameDirectory, relPath) - if err = configFile.Copy(destPath, true); err != nil { - errs = append(errs, errors.Wrapf(err, "Unable to copy file %s", configFile.FilePath())) - } - } else { - // include files could be anywhere - potentially multiple with the same name. We thus, recreate the directory - // hierarchy under includes to avoid collisions - destPath := path.Join(includesDirectory, configFile.FilePath()) - if err = configFile.Copy(destPath, true); err != nil { - errs = append(errs, errors.Wrapf(err, "Unable to copy file %s", configFile.FilePath())) - } - } - - } -} - -func processDirectoryFileFrame(frameId string, frame data.DirectoryFileFrame, baseDir string) ([]string, []error) { - var errs []error - // each set of files goes under its own directory to preserve grouping - frameDirectory := filepath.Join(baseDir, frameId) - if err := os.MkdirAll(frameDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, err) - return []string{frameDirectory}, errs - } - if !ok { - return []string{frameDirectory}, errs - } - file := values[0].(data.SimpleFile) - relPath := strings.TrimPrefix(file.FilePath(), frame.Directory) - destPath := path.Join(frameDirectory, relPath) - - if err = file.Copy(destPath, true); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to copy file %s for frame %s", file, frameId)) - } - } -} - -func processFileFrame(frameId string, frame data.FileFrame, baseDir string) ([]string, []error) { - var errs []error - frameDirectory := filepath.Join(baseDir, frameId) - if err := os.MkdirAll(frameDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, err) - } - if !ok { - return []string{frameDirectory}, errs - } - file := values[0].(data.SimpleFile) - // we need an absolute path to preserve the directory hierarchy - dir, err := filepath.Abs(filepath.Dir(file.FilePath())) - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to determine dir for %s", file.FilePath())) - } - outputDir := filepath.Join(frameDirectory, dir) - if _, err := os.Stat(outputDir); os.IsNotExist(err) { - if err := os.MkdirAll(outputDir, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for %s", file.FilePath())) - } else { - outputPath := filepath.Join(outputDir, filepath.Base(file.FilePath())) - err = file.Copy(outputPath, false) - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to copy file %s", file.FilePath())) - } - } - } - } -} - -func getWorkingDirectory(path string) (string, error) { - if !filepath.IsAbs(path) { - workingPath, err := os.Getwd() - if err != nil { - return "", err - } - return filepath.Join(workingPath, path), nil - } - return path, nil -} - -func createArchiveMap(filePaths []string, prefix string) map[string]string { - archiveMap := make(map[string]string) - for _, path := range filePaths { - archiveMap[path] = strings.TrimPrefix(path, prefix) - } - return archiveMap -} - -func createArchive(outputFile string, filePaths map[string]string) error { - files, err := archiver.FilesFromDisk(nil, filePaths) - if err != nil { - return err - } - out, err := os.Create(outputFile) - if err != nil { - return err - } - defer out.Close() - format := archiver.CompressedArchive{ - Compression: archiver.Gz{}, - Archival: archiver.Tar{}, - } - err = format.Archive(context.Background(), out, files) - return err -} - -func (o SimpleOutput) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "./", - Param: config.NewParam("directory", "Directory in which to create dump. Defaults to the current directory.", false), - }, - config.StringOptions{ - Value: "csv", - // TODO: add tsv and others here later - Options: []string{"csv"}, - Param: config.NewParam("format", "Format of exported files", false), - }, - config.BoolParam{ - Value: false, - Param: config.NewParam("skip_archive", "Don't compress output to an archive", false), - }, - }, - } -} - -func (o SimpleOutput) Description() string { - return "Writes out the diagnostic bundle as files in a structured directory, optionally producing a compressed archive." -} - -// here we register the output for use -func init() { - outputs.Register(OutputName, func() (outputs.Output, error) { - return SimpleOutput{}, nil - }) -} diff --git a/programs/diagnostics/internal/outputs/file/simple_test.go b/programs/diagnostics/internal/outputs/file/simple_test.go deleted file mode 100644 index 471a1c70cc1..00000000000 --- a/programs/diagnostics/internal/outputs/file/simple_test.go +++ /dev/null @@ -1,468 +0,0 @@ -package file_test - -import ( - "bufio" - "encoding/xml" - "fmt" - "io" - "os" - "path" - "strings" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -var clusterFrame = test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - {"events", 2, 1, 1, "dalem-local-clickhouse-blue-2", "192.168.144.4", 9001, 1, "default", "", 0, 0, 0}, - {"events", 3, 1, 1, "dalem-local-clickhouse-blue-3", "192.168.144.3", 9002, 1, "default", "", 0, 0, 0}, - }, -) - -var diskFrame = test.NewFakeDataFrame("disks", []string{"name", "path", "free_space", "total_space", "keep_free_space", "type"}, - [][]interface{}{ - {"default", "/var/lib/clickhouse", 1729659346944, 1938213220352, "", "local"}, - }, -) - -var userFrame = test.NewFakeDataFrame("users", []string{"name", "id", "storage", "auth_type", "auth_params", "host_ip", "host_names", "host_names_regexp", "host_names_like"}, - [][]interface{}{ - {"default", "94309d50-4f52-5250-31bd-74fecac179db,users.xml,plaintext_password", "sha256_password", []string{"::0"}, []string{}, []string{}, []string{}}, - }, -) - -func TestConfiguration(t *testing.T) { - t.Run("correct configuration is returned", func(t *testing.T) { - output := file.SimpleOutput{} - conf := output.Configuration() - require.Len(t, conf.Params, 3) - // check first directory param - require.IsType(t, config.StringParam{}, conf.Params[0]) - directory, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, directory.Required()) - require.Equal(t, "directory", directory.Name()) - require.Equal(t, "./", directory.Value) - // check second format param - require.IsType(t, config.StringOptions{}, conf.Params[1]) - format, ok := conf.Params[1].(config.StringOptions) - require.True(t, ok) - require.False(t, format.Required()) - require.Equal(t, "format", format.Name()) - require.Equal(t, "csv", format.Value) - require.Equal(t, []string{"csv"}, format.Options) - // check third format compress - require.IsType(t, config.BoolParam{}, conf.Params[2]) - skipArchive, ok := conf.Params[2].(config.BoolParam) - require.True(t, ok) - require.False(t, format.Required()) - require.False(t, skipArchive.Value) - }) -} - -func TestWrite(t *testing.T) { - bundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "systemB": { - Frames: map[string]data.Frame{ - "user": userFrame, - }, - }, - } - t.Run("test we can write simple diagnostic sets", func(t *testing.T) { - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", bundles, configuration) - require.Nil(t, err) - require.Equal(t, data.FrameErrors{}, frameErrors) - clusterFile := path.Join(tempDir, "test", "test", "systemA", "cluster.csv") - diskFile := path.Join(tempDir, "test", "test", "systemA", "disk.csv") - userFile := path.Join(tempDir, "test", "test", "systemB", "user.csv") - require.FileExists(t, clusterFile) - require.FileExists(t, diskFile) - require.FileExists(t, userFile) - diskLines, err := readFileLines(diskFile) - require.Nil(t, err) - require.Len(t, diskLines, 2) - usersLines, err := readFileLines(userFile) - require.Nil(t, err) - require.Len(t, usersLines, 2) - clusterLines, err := readFileLines(clusterFile) - require.Nil(t, err) - require.Len(t, clusterLines, 4) - require.Equal(t, strings.Join(clusterFrame.ColumnNames, ","), clusterLines[0]) - require.Equal(t, "events,1,1,1,dalem-local-clickhouse-blue-1,192.168.144.2,9000,1,default,,0,0,0", clusterLines[1]) - require.Equal(t, "events,2,1,1,dalem-local-clickhouse-blue-2,192.168.144.4,9001,1,default,,0,0,0", clusterLines[2]) - require.Equal(t, "events,3,1,1,dalem-local-clickhouse-blue-3,192.168.144.3,9002,1,default,,0,0,0", clusterLines[3]) - resetFrames() - }) - - t.Run("test invalid parameter", func(t *testing.T) { - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - config.StringOptions{ - Value: "random", - Options: []string{"csv"}, - // TODO: add tsv and others here later - Param: config.NewParam("format", "Format of exported files", false), - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip compressed archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", bundles, configuration) - require.Equal(t, data.FrameErrors{}, frameErrors) - require.NotNil(t, err) - require.Equal(t, "parameter format is invalid - random is not a valid value for format - [csv]", err.Error()) - resetFrames() - }) - - t.Run("test compression", func(t *testing.T) { - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", bundles, configuration) - require.Nil(t, err) - require.Equal(t, data.FrameErrors{}, frameErrors) - archiveFileName := path.Join(tempDir, "test", "test.tar.gz") - fi, err := os.Stat(archiveFileName) - require.Nil(t, err) - require.FileExists(t, archiveFileName) - // compression will vary so lets test range - require.Greater(t, int64(600), fi.Size()) - require.Less(t, int64(200), fi.Size()) - outputFolder := path.Join(tempDir, "test", "test") - // check the folder doesn't exist and is cleaned up - require.NoFileExists(t, outputFolder) - resetFrames() - }) - - t.Run("test support for directory frames", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - files := createRandomFiles(tempDir, 5) - dirFrame, errs := data.NewFileDirectoryFrame(tempDir, []string{"*.log"}) - require.Empty(t, errs) - fileBundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "config": { - Frames: map[string]data.Frame{ - "logs": dirFrame, - }, - }, - } - destDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: destDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", fileBundles, configuration) - require.Nil(t, err) - require.NotNil(t, frameErrors) - - // test the usual frames still work - clusterFile := path.Join(destDir, "test", "test", "systemA", "cluster.csv") - diskFile := path.Join(destDir, "test", "test", "systemA", "disk.csv") - require.FileExists(t, clusterFile) - require.FileExists(t, diskFile) - diskLines, err := readFileLines(diskFile) - require.Nil(t, err) - require.Len(t, diskLines, 2) - clusterLines, err := readFileLines(clusterFile) - require.Nil(t, err) - require.Len(t, clusterLines, 4) - require.Equal(t, strings.Join(clusterFrame.ColumnNames, ","), clusterLines[0]) - require.Equal(t, "events,1,1,1,dalem-local-clickhouse-blue-1,192.168.144.2,9000,1,default,,0,0,0", clusterLines[1]) - require.Equal(t, "events,2,1,1,dalem-local-clickhouse-blue-2,192.168.144.4,9001,1,default,,0,0,0", clusterLines[2]) - require.Equal(t, "events,3,1,1,dalem-local-clickhouse-blue-3,192.168.144.3,9002,1,default,,0,0,0", clusterLines[3]) - //test our directory frame - for _, filepath := range files { - // check they were copied - subPath := strings.TrimPrefix(filepath, tempDir) - // path here will be //test>/config/logs/ - newPath := path.Join(destDir, "test", "test", "config", "logs", subPath) - require.FileExists(t, newPath) - } - resetFrames() - }) - - t.Run("test support for config frames", func(t *testing.T) { - xmlConfig := data.XmlConfig{ - XMLName: xml.Name{}, - Clickhouse: data.XmlLoggerConfig{ - XMLName: xml.Name{}, - ErrorLog: "/var/log/clickhouse-server/clickhouse-server.err.log", - Log: "/var/log/clickhouse-server/clickhouse-server.log", - }, - IncludeFrom: "", - } - tempDir := t.TempDir() - confDir := path.Join(tempDir, "conf") - // create an includes file - includesDir := path.Join(tempDir, "includes") - err := os.MkdirAll(includesDir, os.ModePerm) - require.Nil(t, err) - includesPath := path.Join(includesDir, "random.xml") - includesFile, err := os.Create(includesPath) - require.Nil(t, err) - xmlWriter := io.Writer(includesFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - err = enc.Encode(xmlConfig) - require.Nil(t, err) - // create 5 temporary config files - files := make([]string, 5) - // set the includes - xmlConfig.IncludeFrom = includesPath - for i := 0; i < 5; i++ { - // we want to check hierarchies are preserved so create a simple folder for each file - fileDir := path.Join(confDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.xml", i)) - files[i] = filepath - xmlFile, err := os.Create(filepath) - require.Nil(t, err) - // write a little xml so its valid - xmlWriter := io.Writer(xmlFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - err = enc.Encode(xmlConfig) - require.Nil(t, err) - } - configFrame, errs := data.NewConfigFileFrame(confDir) - require.Empty(t, errs) - fileBundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "config": { - Frames: map[string]data.Frame{ - "user_specified": configFrame, - }, - }, - } - destDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: destDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", fileBundles, configuration) - require.Nil(t, err) - require.NotNil(t, frameErrors) - require.Empty(t, frameErrors.Errors) - //test our config frame - for _, filepath := range files { - // check they were copied - subPath := strings.TrimPrefix(filepath, confDir) - // path here will be //test>/config/user_specified/file - newPath := path.Join(destDir, "test", "test", "config", "user_specified", subPath) - require.FileExists(t, newPath) - } - // check our includes file exits - // path here will be //test>/config/user_specified/file/includes - require.FileExists(t, path.Join(destDir, "test", "test", "config", "user_specified", "includes", includesPath)) - resetFrames() - }) - - t.Run("test support for file frames", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - files := createRandomFiles(tempDir, 5) - fileFrame := data.NewFileFrame("collection", files) - fileBundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "file": { - Frames: map[string]data.Frame{ - "collection": fileFrame, - }, - }, - } - destDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: destDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", fileBundles, configuration) - require.Nil(t, err) - require.NotNil(t, frameErrors) - //test our directory frame - for _, filepath := range files { - // path here will be //test>/file/collection/ - newPath := path.Join(destDir, "test", "test", "file", "collection", filepath) - require.FileExists(t, newPath) - } - resetFrames() - }) - - t.Run("test support for hierarchical frames", func(t *testing.T) { - bottomFrame := data.NewHierarchicalFrame("bottomLevel", userFrame, []data.HierarchicalFrame{}) - middleFrame := data.NewHierarchicalFrame("middleLevel", diskFrame, []data.HierarchicalFrame{bottomFrame}) - topFrame := data.NewHierarchicalFrame("topLevel", clusterFrame, []data.HierarchicalFrame{middleFrame}) - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - hierarchicalBundle := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "topLevel": topFrame, - }, - }, - } - frameErrors, err := output.Write("test", hierarchicalBundle, configuration) - require.Nil(t, err) - require.Equal(t, data.FrameErrors{}, frameErrors) - topFile := path.Join(tempDir, "test", "test", "systemA", "topLevel.csv") - middleFile := path.Join(tempDir, "test", "test", "systemA", "middleLevel", "middleLevel.csv") - bottomFile := path.Join(tempDir, "test", "test", "systemA", "middleLevel", "bottomLevel", "bottomLevel.csv") - require.FileExists(t, topFile) - require.FileExists(t, middleFile) - require.FileExists(t, bottomFile) - topLines, err := readFileLines(topFile) - require.Nil(t, err) - require.Len(t, topLines, 4) - middleLines, err := readFileLines(middleFile) - require.Nil(t, err) - require.Len(t, middleLines, 2) - bottomLines, err := readFileLines(bottomFile) - require.Nil(t, err) - require.Len(t, bottomLines, 2) - require.Equal(t, strings.Join(clusterFrame.ColumnNames, ","), topLines[0]) - require.Equal(t, "events,1,1,1,dalem-local-clickhouse-blue-1,192.168.144.2,9000,1,default,,0,0,0", topLines[1]) - require.Equal(t, "events,2,1,1,dalem-local-clickhouse-blue-2,192.168.144.4,9001,1,default,,0,0,0", topLines[2]) - require.Equal(t, "events,3,1,1,dalem-local-clickhouse-blue-3,192.168.144.3,9002,1,default,,0,0,0", topLines[3]) - resetFrames() - }) -} - -func createRandomFiles(tempDir string, num int) []string { - files := make([]string, num) - for i := 0; i < 5; i++ { - // we want to check hierarchies are preserved so create a simple folder for each file - fileDir := path.Join(tempDir, fmt.Sprintf("%d", i)) - os.MkdirAll(fileDir, os.ModePerm) //nolint:errcheck - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.log", i)) - files[i] = filepath - os.Create(filepath) //nolint:errcheck - } - return files -} - -func resetFrames() { - clusterFrame.Reset() - userFrame.Reset() - diskFrame.Reset() -} - -func readFileLines(filename string) ([]string, error) { - file, err := os.Open(filename) - if err != nil { - return nil, err - } - defer file.Close() - - var lines []string - scanner := bufio.NewScanner(file) - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - return lines, scanner.Err() -} - -func staticFolderName() string { - return "test" -} diff --git a/programs/diagnostics/internal/outputs/registry.go b/programs/diagnostics/internal/outputs/registry.go deleted file mode 100644 index 0187cd9105d..00000000000 --- a/programs/diagnostics/internal/outputs/registry.go +++ /dev/null @@ -1,67 +0,0 @@ -package outputs - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -type Output interface { - Write(id string, bundles map[string]*data.DiagnosticBundle, config config.Configuration) (data.FrameErrors, error) - Configuration() config.Configuration - Description() string - // TODO: we will need to implement this for the convert function - //Read(config config.Configuration) (data.DiagnosticBundle, error) -} - -// Register can be called from init() on an output in this package -// It will automatically be added to the Outputs map to be called externally -func Register(name string, output OutputFactory) { - // names must be unique - if _, ok := Outputs[name]; ok { - log.Error().Msgf("More than 1 output is trying to register under the name %s. Names must be unique.", name) - } - Outputs[name] = output -} - -// OutputFactory lets us use a closure to get instances of the output struct -type OutputFactory func() (Output, error) - -var Outputs = map[string]OutputFactory{} - -func GetOutputNames() []string { - outputs := make([]string, len(Outputs)) - i := 0 - for k := range Outputs { - outputs[i] = k - i++ - } - return outputs -} - -func GetOutputByName(name string) (Output, error) { - if outputFactory, ok := Outputs[name]; ok { - //do something here - output, err := outputFactory() - if err != nil { - return nil, errors.Wrapf(err, "output %s could not be initialized", name) - } - return output, nil - } - return nil, fmt.Errorf("%s is not a valid output name", name) -} - -func BuildConfigurationOptions() (map[string]config.Configuration, error) { - configurations := make(map[string]config.Configuration) - for name, collectorFactory := range Outputs { - output, err := collectorFactory() - if err != nil { - return nil, errors.Wrapf(err, "output %s could not be initialized", name) - } - configurations[name] = output.Configuration() - } - return configurations, nil -} diff --git a/programs/diagnostics/internal/outputs/registry_test.go b/programs/diagnostics/internal/outputs/registry_test.go deleted file mode 100644 index ba8408e5a59..00000000000 --- a/programs/diagnostics/internal/outputs/registry_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package outputs_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" - "github.com/stretchr/testify/require" -) - -func TestGetOutputNames(t *testing.T) { - t.Run("can get all output names", func(t *testing.T) { - outputNames := outputs.GetOutputNames() - require.ElementsMatch(t, []string{"simple", "report"}, outputNames) - }) - -} - -func TestGetOutputByName(t *testing.T) { - - t.Run("can get output by name", func(t *testing.T) { - output, err := outputs.GetOutputByName("simple") - require.Nil(t, err) - require.Equal(t, file.SimpleOutput{}, output) - }) - - t.Run("fails on non existing output", func(t *testing.T) { - output, err := outputs.GetOutputByName("random") - require.NotNil(t, err) - require.Equal(t, "random is not a valid output name", err.Error()) - require.Nil(t, output) - }) -} - -func TestBuildConfigurationOptions(t *testing.T) { - - t.Run("can get all output configurations", func(t *testing.T) { - outputs, err := outputs.BuildConfigurationOptions() - require.Nil(t, err) - require.Len(t, outputs, 2) - require.Contains(t, outputs, "simple") - require.Contains(t, outputs, "report") - }) -} diff --git a/programs/diagnostics/internal/outputs/terminal/report.go b/programs/diagnostics/internal/outputs/terminal/report.go deleted file mode 100644 index 8337f542457..00000000000 --- a/programs/diagnostics/internal/outputs/terminal/report.go +++ /dev/null @@ -1,284 +0,0 @@ -package terminal - -import ( - "bufio" - "fmt" - "os" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/olekukonko/tablewriter" - "github.com/pkg/errors" -) - -const OutputName = "report" - -type ReportOutput struct { -} - -func (r ReportOutput) Write(id string, bundles map[string]*data.DiagnosticBundle, conf config.Configuration) (data.FrameErrors, error) { - conf, err := conf.ValidateConfig(r.Configuration()) - if err != nil { - return data.FrameErrors{}, err - } - format, err := config.ReadStringOptionsValue(conf, "format") - if err != nil { - return data.FrameErrors{}, err - } - nonInteractive, err := config.ReadBoolValue(conf, "continue") - if err != nil { - return data.FrameErrors{}, err - } - maxRows, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return data.FrameErrors{}, err - } - maxColumns, err := config.ReadIntValue(conf, "column_limit") - if err != nil { - return data.FrameErrors{}, err - } - frameErrors := data.FrameErrors{} - for name := range bundles { - frameError := printDiagnosticBundle(name, bundles[name], format, !nonInteractive, int(maxRows), int(maxColumns)) - frameErrors.Errors = append(frameErrors.Errors, frameError.Errors...) - } - return data.FrameErrors{}, nil -} - -func printDiagnosticBundle(name string, diag *data.DiagnosticBundle, format string, interactive bool, maxRows, maxColumns int) data.FrameErrors { - frameErrors := data.FrameErrors{} - for frameId, frame := range diag.Frames { - printFrameHeader(fmt.Sprintf("%s.%s", name, frameId)) - err := printFrame(frame, format, maxRows, maxColumns) - if err != nil { - frameErrors.Errors = append(frameErrors.Errors, err) - } - if interactive { - err := waitForEnter() - if err != nil { - frameErrors.Errors = append(frameErrors.Errors, err) - } - } - } - return frameErrors -} - -func waitForEnter() error { - fmt.Println("Press the Enter Key to view the next frame report") - for { - consoleReader := bufio.NewReaderSize(os.Stdin, 1) - input, err := consoleReader.ReadByte() - if err != nil { - return errors.New("Unable to read user input") - } - if input == 3 { - //ctl +c - fmt.Println("Exiting...") - os.Exit(0) - } - if input == 10 { - return nil - } - } -} - -func printFrame(frame data.Frame, format string, maxRows, maxColumns int) error { - switch f := frame.(type) { - case data.DatabaseFrame: - return printDatabaseFrame(f, format, maxRows, maxColumns) - case data.ConfigFileFrame: - return printConfigFrame(f, format) - case data.DirectoryFileFrame: - return printDirectoryFileFrame(f, format, maxRows) - case data.HierarchicalFrame: - return printHierarchicalFrame(f, format, maxRows, maxColumns) - default: - // for now our data frame writer supports all frames - return printDatabaseFrame(f, format, maxRows, maxColumns) - } -} - -func createTable(format string) *tablewriter.Table { - table := tablewriter.NewWriter(os.Stdout) - if format == "markdown" { - table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) - table.SetCenterSeparator("|") - } - return table -} - -func printFrameHeader(title string) { - titleTable := tablewriter.NewWriter(os.Stdout) - titleTable.SetHeader([]string{title}) - titleTable.SetAutoWrapText(false) - titleTable.SetAutoFormatHeaders(true) - titleTable.SetHeaderAlignment(tablewriter.ALIGN_CENTER) - titleTable.SetRowSeparator("\n") - titleTable.SetHeaderLine(false) - titleTable.SetBorder(false) - titleTable.SetTablePadding("\t") // pad with tabs - titleTable.SetNoWhiteSpace(true) - titleTable.Render() -} - -func printHierarchicalFrame(frame data.HierarchicalFrame, format string, maxRows, maxColumns int) error { - err := printDatabaseFrame(frame, format, maxRows, maxColumns) - if err != nil { - return err - } - for _, subFrame := range frame.SubFrames { - err = printHierarchicalFrame(subFrame, format, maxRows, maxColumns) - if err != nil { - return err - } - } - return nil -} - -func printDatabaseFrame(frame data.Frame, format string, maxRows, maxColumns int) error { - table := createTable(format) - table.SetAutoWrapText(false) - columns := len(frame.Columns()) - if maxColumns > 0 && maxColumns < columns { - columns = maxColumns - } - table.SetHeader(frame.Columns()[:columns]) - r := 0 - trunColumns := 0 - for { - values, ok, err := frame.Next() - if !ok || r == maxRows { - table.Render() - if trunColumns > 0 { - warning(fmt.Sprintf("Truncated %d columns, more available...", trunColumns)) - } - if r == maxRows { - warning("Truncated rows, more available...") - } - return err - } - if err != nil { - return err - } - columns := len(values) - // -1 means unlimited - if maxColumns > 0 && maxColumns < columns { - trunColumns = columns - maxColumns - columns = maxColumns - } - row := make([]string, columns) - for i, value := range values { - if i == columns { - break - } - row[i] = fmt.Sprintf("%v", value) - } - table.Append(row) - r++ - } -} - -// currently we dump the whole config - useless in parts -func printConfigFrame(frame data.Frame, format string) error { - for { - values, ok, err := frame.Next() - if !ok { - return err - } - if err != nil { - return err - } - configFile := values[0].(data.File) - dat, err := os.ReadFile(configFile.FilePath()) - if err != nil { - return err - } - // create a table per row - as each will be a file - table := createTable(format) - table.SetAutoWrapText(false) - table.SetAutoFormatHeaders(false) - table.ClearRows() - table.SetHeader([]string{configFile.FilePath()}) - table.Append([]string{string(dat)}) - table.Render() - } -} - -func printDirectoryFileFrame(frame data.Frame, format string, maxRows int) error { - for { - values, ok, err := frame.Next() - if !ok { - - return err - } - if err != nil { - return err - } - path := values[0].(data.SimpleFile) - file, err := os.Open(path.FilePath()) - if err != nil { - // failure on one file causes rest to be ignored in frame...we could improve this - return errors.Wrapf(err, "Unable to read file %s", path.FilePath()) - } - scanner := bufio.NewScanner(file) - i := 0 - // create a table per row - as each will be a file - table := createTable(format) - table.SetAutoWrapText(false) - table.SetAutoFormatHeaders(false) - table.ClearRows() - table.SetHeader([]string{path.FilePath()}) - for scanner.Scan() { - if i == maxRows { - fmt.Println() - table.Render() - warning("Truncated lines, more available...") - fmt.Print("\n") - break - } - table.Append([]string{scanner.Text()}) - i++ - } - } -} - -// prints a warning -func warning(s string) { - fmt.Printf("\x1b[%dm%v\x1b[0m%s\n", 33, "WARNING: ", s) -} - -func (r ReportOutput) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringOptions{ - Value: "default", - Options: []string{"default", "markdown"}, - Param: config.NewParam("format", "Format of tables. Default is terminal friendly.", false), - }, - config.BoolParam{ - Value: false, - Param: config.NewParam("continue", "Print report with no interaction", false), - }, - config.IntParam{ - Value: 10, - Param: config.NewParam("row_limit", "Max Rows to print per frame.", false), - }, - config.IntParam{ - Value: 8, - Param: config.NewParam("column_limit", "Max Columns to print per frame. Negative is unlimited.", false), - }, - }, - } -} - -func (r ReportOutput) Description() string { - return "Writes out the diagnostic bundle to the terminal as a simple report." -} - -// here we register the output for use -func init() { - outputs.Register(OutputName, func() (outputs.Output, error) { - return ReportOutput{}, nil - }) -} diff --git a/programs/diagnostics/internal/platform/config/models.go b/programs/diagnostics/internal/platform/config/models.go deleted file mode 100644 index 6c76b8f149b..00000000000 --- a/programs/diagnostics/internal/platform/config/models.go +++ /dev/null @@ -1,129 +0,0 @@ -package config - -import ( - "fmt" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" -) - -type ConfigParam interface { - Name() string - Required() bool - Description() string - validate(defaultConfig ConfigParam) error -} - -type Configuration struct { - Params []ConfigParam -} - -type Param struct { - name string - description string - required bool -} - -func NewParam(name string, description string, required bool) Param { - return Param{ - name: name, - description: description, - required: required, - } -} - -func (bp Param) Name() string { - return bp.name -} - -func (bp Param) Required() bool { - return bp.required -} - -func (bp Param) Description() string { - return bp.description -} - -func (bp Param) validate(defaultConfig ConfigParam) error { - return nil -} - -func (c Configuration) GetConfigParam(paramName string) (ConfigParam, error) { - for _, param := range c.Params { - if param.Name() == paramName { - return param, nil - } - } - return nil, fmt.Errorf("%s does not exist", paramName) -} - -// ValidateConfig finds the intersection of a config c and a default config. Requires all possible params to be in default. -func (c Configuration) ValidateConfig(defaultConfig Configuration) (Configuration, error) { - var finalParams []ConfigParam - for _, defaultParam := range defaultConfig.Params { - setParam, err := c.GetConfigParam(defaultParam.Name()) - if err == nil { - // check the set value is valid - if err := setParam.validate(defaultParam); err != nil { - return Configuration{}, fmt.Errorf("parameter %s is invalid - %s", defaultParam.Name(), err.Error()) - } - finalParams = append(finalParams, setParam) - } else if defaultParam.Required() { - return Configuration{}, fmt.Errorf("missing required parameter %s - %s", defaultParam.Name(), err.Error()) - } else { - finalParams = append(finalParams, defaultParam) - } - } - return Configuration{ - Params: finalParams, - }, nil -} - -type StringParam struct { - Param - Value string - AllowEmpty bool -} - -func (sp StringParam) validate(defaultConfig ConfigParam) error { - dsp := defaultConfig.(StringParam) - if !dsp.AllowEmpty && strings.TrimSpace(sp.Value) == "" { - return fmt.Errorf("%s cannot be empty", sp.Name()) - } - // if the parameter is not required it doesn't matter - return nil -} - -type StringListParam struct { - Param - Values []string -} - -type StringOptions struct { - Param - Options []string - Value string - AllowEmpty bool -} - -func (so StringOptions) validate(defaultConfig ConfigParam) error { - dso := defaultConfig.(StringOptions) - if !dso.AllowEmpty && strings.TrimSpace(so.Value) == "" { - return fmt.Errorf("%s cannot be empty", so.Name()) - } - if !utils.Contains(dso.Options, so.Value) { - return fmt.Errorf("%s is not a valid value for %s - %v", so.Value, so.Name(), so.Options) - } - // if the parameter is not required it doesn't matter - return nil -} - -type IntParam struct { - Param - Value int64 -} - -type BoolParam struct { - Param - Value bool -} diff --git a/programs/diagnostics/internal/platform/config/models_test.go b/programs/diagnostics/internal/platform/config/models_test.go deleted file mode 100644 index 916d20ec28b..00000000000 --- a/programs/diagnostics/internal/platform/config/models_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package config_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/stretchr/testify/require" -) - -var conf = config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - Values: []string{"some", "values"}, - Param: config.NewParam("paramA", "", false), - }, - config.StringParam{ - Value: "random", - Param: config.NewParam("paramB", "", true), - }, - config.StringParam{ - Value: "", - AllowEmpty: true, - Param: config.NewParam("paramC", "", false), - }, - config.StringOptions{ - Value: "random", - Options: []string{"random", "very_random", "very_very_random"}, - Param: config.NewParam("paramD", "", false), - AllowEmpty: true, - }, - }, -} - -func TestGetConfigParam(t *testing.T) { - - t.Run("can find get config param by name", func(t *testing.T) { - paramA, err := conf.GetConfigParam("paramA") - require.Nil(t, err) - require.NotNil(t, paramA) - require.IsType(t, config.StringListParam{}, paramA) - stringListParam, ok := paramA.(config.StringListParam) - require.True(t, ok) - require.False(t, stringListParam.Required()) - require.Equal(t, stringListParam.Name(), "paramA") - require.ElementsMatch(t, stringListParam.Values, []string{"some", "values"}) - }) - - t.Run("throws error on missing element", func(t *testing.T) { - paramZ, err := conf.GetConfigParam("paramZ") - require.Nil(t, paramZ) - require.NotNil(t, err) - require.Equal(t, err.Error(), "paramZ does not exist") - }) -} - -func TestValidateConfig(t *testing.T) { - - t.Run("validate adds the default and allows override", func(t *testing.T) { - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "custom", - Param: config.NewParam("paramB", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, err) - require.NotNil(t, newConf) - require.Len(t, newConf.Params, 4) - // check first param - require.IsType(t, config.StringListParam{}, newConf.Params[0]) - stringListParam, ok := newConf.Params[0].(config.StringListParam) - require.True(t, ok) - require.False(t, stringListParam.Required()) - require.Equal(t, stringListParam.Name(), "paramA") - require.ElementsMatch(t, stringListParam.Values, []string{"some", "values"}) - // check second param - require.IsType(t, config.StringParam{}, newConf.Params[1]) - stringParam, ok := newConf.Params[1].(config.StringParam) - require.True(t, ok) - require.True(t, stringParam.Required()) - require.Equal(t, "paramB", stringParam.Name()) - require.Equal(t, "custom", stringParam.Value) - }) - - t.Run("validate errors if missing param", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - Values: []string{"some", "values"}, - Param: config.NewParam("paramA", "", false), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, newConf.Params) - require.NotNil(t, err) - require.Equal(t, "missing required parameter paramB - paramB does not exist", err.Error()) - }) - - t.Run("validate errors if invalid string value", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("paramB", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, newConf.Params) - require.NotNil(t, err) - require.Equal(t, "parameter paramB is invalid - paramB cannot be empty", err.Error()) - }) - - t.Run("allow empty string value if specified", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("paramC", "", true), - }, - config.StringParam{ - Value: "custom", - Param: config.NewParam("paramB", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.NotNil(t, newConf.Params) - require.Nil(t, err) - }) - - t.Run("validate errors if invalid string options value", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "not_random", - Param: config.NewParam("paramB", "", true), - }, - config.StringOptions{ - Value: "custom", - // this isn't ideal we need to ensure options are set for this to validate correctly - Options: []string{"random", "very_random", "very_very_random"}, - Param: config.NewParam("paramD", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, newConf.Params) - require.NotNil(t, err) - require.Equal(t, "parameter paramD is invalid - custom is not a valid value for paramD - [random very_random very_very_random]", err.Error()) - }) - - t.Run("allow empty string value for StringOptions if specified", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "custom", - Param: config.NewParam("paramB", "", true), - }, - config.StringOptions{ - Param: config.Param{}, - // this isn't ideal we need to ensure options are set for this to validate correctly - Options: []string{"random", "very_random", "very_very_random"}, - Value: "", - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.NotNil(t, newConf.Params) - require.Nil(t, err) - }) - - //TODO: Do we need to test if parameters of the same name but wrong type are passed?? -} diff --git a/programs/diagnostics/internal/platform/config/utils.go b/programs/diagnostics/internal/platform/config/utils.go deleted file mode 100644 index 5f84c38d4f4..00000000000 --- a/programs/diagnostics/internal/platform/config/utils.go +++ /dev/null @@ -1,74 +0,0 @@ -package config - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" -) - -func ReadStringListValues(conf Configuration, paramName string) ([]string, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return nil, err - } - value, ok := param.(StringListParam) - if !ok { - value, ok = param.(StringListParam) - if !ok { - return nil, fmt.Errorf("%s must be a list of strings", paramName) - } - } - - return value.Values, nil -} - -func ReadStringValue(conf Configuration, paramName string) (string, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return "", err - } - value, ok := param.(StringParam) - if !ok { - return "", fmt.Errorf("%s must be a list of strings", paramName) - } - return value.Value, nil -} - -func ReadIntValue(conf Configuration, paramName string) (int64, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return 0, err - } - value, ok := param.(IntParam) - if !ok { - return 9, fmt.Errorf("%s must be an unsigned integer", paramName) - } - return value.Value, nil -} - -func ReadBoolValue(conf Configuration, paramName string) (bool, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return false, err - } - value, ok := param.(BoolParam) - if !ok { - return false, fmt.Errorf("%s must be a boolean", paramName) - } - return value.Value, nil -} - -func ReadStringOptionsValue(conf Configuration, paramName string) (string, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return "", err - } - value, ok := param.(StringOptions) - if !ok { - return "", fmt.Errorf("%s must be a string options", paramName) - } - if !utils.Contains(value.Options, value.Value) { - return "", fmt.Errorf("%s is not a valid option in %v for the the parameter %s", value.Value, value.Options, paramName) - } - return value.Value, nil -} diff --git a/programs/diagnostics/internal/platform/config/utils_test.go b/programs/diagnostics/internal/platform/config/utils_test.go deleted file mode 100644 index 9e03e5e69d2..00000000000 --- a/programs/diagnostics/internal/platform/config/utils_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package config_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/stretchr/testify/require" -) - -func TestReadStringListValues(t *testing.T) { - - t.Run("can find a string list param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringListParam{ - Values: []string{"licenses", "settings"}, - Param: config.NewParam("exclude_tables", "Specify list of tables not to collect", false), - }, - }, - } - excludeTables, err := config.ReadStringListValues(conf, "exclude_tables") - require.Nil(t, err) - require.Equal(t, []string{"licenses", "settings"}, excludeTables) - }) - -} - -func TestReadStringValue(t *testing.T) { - - t.Run("can find a string param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringParam{ - Value: "/tmp/dump", - Param: config.NewParam("directory", "Specify a directory", false), - }, - }, - } - directory, err := config.ReadStringValue(conf, "directory") - require.Nil(t, err) - require.Equal(t, "/tmp/dump", directory) - }) - -} - -func TestReadIntValue(t *testing.T) { - t.Run("can find an integer param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - // nil means include everything - Value: 10000, - Param: config.NewParam("row_limit", "Max Rows to collect", false), - }, - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringParam{ - Value: "/tmp/dump", - Param: config.NewParam("directory", "Specify a directory", false), - }, - }, - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - require.Nil(t, err) - require.Equal(t, int64(10000), rowLimit) - }) - -} - -func TestReadBoolValue(t *testing.T) { - t.Run("can find a boolean param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.BoolParam{ - // nil means include everything - Value: true, - Param: config.NewParam("compress", "Compress data", false), - }, - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringParam{ - Value: "/tmp/dump", - Param: config.NewParam("directory", "Specify a directory", false), - }, - }, - } - - compress, err := config.ReadBoolValue(conf, "compress") - require.Nil(t, err) - require.True(t, compress) - }) -} - -func TestReadStringOptionsValue(t *testing.T) { - t.Run("can find a string value in a list of options", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringOptions{ - Param: config.NewParam("format", "List of formats", false), - Options: []string{"csv", "tsv", "binary", "json", "ndjson"}, - Value: "csv", - AllowEmpty: false, - }, - }, - } - format, err := config.ReadStringOptionsValue(conf, "format") - require.Nil(t, err) - require.Equal(t, "csv", format) - }) - - t.Run("errors on invalid value", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringOptions{ - Param: config.NewParam("format", "List of formats", false), - Options: []string{"csv", "tsv", "binary", "json", "ndjson"}, - Value: "random", - AllowEmpty: false, - }, - }, - } - format, err := config.ReadStringOptionsValue(conf, "format") - require.Equal(t, "random is not a valid option in [csv tsv binary json ndjson] for the the parameter format", err.Error()) - require.Equal(t, "", format) - }) -} diff --git a/programs/diagnostics/internal/platform/data/bundle.go b/programs/diagnostics/internal/platform/data/bundle.go deleted file mode 100644 index e4eeede659e..00000000000 --- a/programs/diagnostics/internal/platform/data/bundle.go +++ /dev/null @@ -1,27 +0,0 @@ -package data - -import ( - "strings" -) - -// DiagnosticBundle contains the results from a Collector -// each frame can represent a table or collection of data files. By allowing multiple frames a single DiagnosticBundle -// can potentially contain many related tables -type DiagnosticBundle struct { - Frames map[string]Frame - // Errors is a property to be set if the Collector has an error. This can be used to indicate a partial collection - // and failed frames - Errors FrameErrors -} - -type FrameErrors struct { - Errors []error -} - -func (fe *FrameErrors) Error() string { - errors := make([]string, len(fe.Errors)) - for i := range errors { - errors[i] = fe.Errors[i].Error() - } - return strings.Join(errors, "\n") -} diff --git a/programs/diagnostics/internal/platform/data/bundle_test.go b/programs/diagnostics/internal/platform/data/bundle_test.go deleted file mode 100644 index ff9cfc2cf56..00000000000 --- a/programs/diagnostics/internal/platform/data/bundle_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package data_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/stretchr/testify/require" -) - -func TestBundleError(t *testing.T) { - - t.Run("can get a bundle error", func(t *testing.T) { - errs := make([]error, 3) - errs[0] = errors.New("Error 1") - errs[1] = errors.New("Error 2") - errs[2] = errors.New("Error 3") - fErrors := data.FrameErrors{ - Errors: errs, - } - require.Equal(t, `Error 1 -Error 2 -Error 3`, fErrors.Error()) - - }) -} diff --git a/programs/diagnostics/internal/platform/data/database.go b/programs/diagnostics/internal/platform/data/database.go deleted file mode 100644 index d49317d8719..00000000000 --- a/programs/diagnostics/internal/platform/data/database.go +++ /dev/null @@ -1,88 +0,0 @@ -package data - -import ( - "database/sql" - "fmt" - "reflect" - "strings" -) - -type DatabaseFrame struct { - name string - ColumnNames []string - rows *sql.Rows - columnTypes []*sql.ColumnType - vars []interface{} -} - -func NewDatabaseFrame(name string, rows *sql.Rows) (DatabaseFrame, error) { - databaseFrame := DatabaseFrame{} - columnTypes, err := rows.ColumnTypes() - if err != nil { - return DatabaseFrame{}, err - } - databaseFrame.columnTypes = columnTypes - databaseFrame.name = name - vars := make([]interface{}, len(columnTypes)) - columnNames := make([]string, len(columnTypes)) - for i := range columnTypes { - value := reflect.Zero(columnTypes[i].ScanType()).Interface() - vars[i] = &value - columnNames[i] = columnTypes[i].Name() - } - databaseFrame.ColumnNames = columnNames - databaseFrame.vars = vars - databaseFrame.rows = rows - return databaseFrame, nil -} - -func (f DatabaseFrame) Next() ([]interface{}, bool, error) { - values := make([]interface{}, len(f.columnTypes)) - for f.rows.Next() { - if err := f.rows.Scan(f.vars...); err != nil { - return nil, false, err - } - for i := range f.columnTypes { - ptr := reflect.ValueOf(f.vars[i]) - values[i] = ptr.Elem().Interface() - } - return values, true, nil //nolint - } - // TODO: raise issue as this seems to always raise an error - //err := f.rows.Err() - f.rows.Close() - return nil, false, nil -} - -func (f DatabaseFrame) Columns() []string { - return f.ColumnNames -} - -func (f DatabaseFrame) Name() string { - return f.name -} - -type Order int - -const ( - Asc Order = 1 - Desc Order = 2 -) - -type OrderBy struct { - Column string - Order Order -} - -func (o OrderBy) String() string { - if strings.TrimSpace(o.Column) == "" { - return "" - } - switch o.Order { - case Asc: - return fmt.Sprintf(" ORDER BY %s ASC", o.Column) - case Desc: - return fmt.Sprintf(" ORDER BY %s DESC", o.Column) - } - return "" -} diff --git a/programs/diagnostics/internal/platform/data/database_test.go b/programs/diagnostics/internal/platform/data/database_test.go deleted file mode 100644 index 57d89e78efc..00000000000 --- a/programs/diagnostics/internal/platform/data/database_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package data_test - -import ( - "database/sql" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/DATA-DOG/go-sqlmock" - "github.com/stretchr/testify/require" -) - -func TestString(t *testing.T) { - t.Run("can order by asc", func(t *testing.T) { - orderBy := data.OrderBy{ - Column: "created_at", - Order: data.Asc, - } - require.Equal(t, " ORDER BY created_at ASC", orderBy.String()) - }) - - t.Run("can order by desc", func(t *testing.T) { - orderBy := data.OrderBy{ - Column: "created_at", - Order: data.Desc, - } - require.Equal(t, " ORDER BY created_at DESC", orderBy.String()) - }) - -} - -func TestNextDatabaseFrame(t *testing.T) { - - t.Run("can iterate sql rows", func(t *testing.T) { - rowValues := [][]interface{}{ - {int64(1), "post_1", "hello"}, - {int64(2), "post_2", "world"}, - {int64(3), "post_3", "goodbye"}, - {int64(4), "post_4", "world"}, - } - mockRows := sqlmock.NewRows([]string{"id", "title", "body"}) - for i := range rowValues { - mockRows.AddRow(rowValues[i][0], rowValues[i][1], rowValues[i][2]) - } - rows := mockRowsToSqlRows(mockRows) - dbFrame, err := data.NewDatabaseFrame("test", rows) - require.ElementsMatch(t, dbFrame.Columns(), []string{"id", "title", "body"}) - require.Nil(t, err) - i := 0 - for { - values, ok, err := dbFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 3) - require.ElementsMatch(t, values, rowValues[i]) - i++ - } - require.Equal(t, 4, i) - }) - - t.Run("can iterate empty sql rows", func(t *testing.T) { - mockRows := sqlmock.NewRows([]string{"id", "title", "body"}) - rows := mockRowsToSqlRows(mockRows) - dbFrame, err := data.NewDatabaseFrame("test", rows) - require.ElementsMatch(t, dbFrame.Columns(), []string{"id", "title", "body"}) - require.Nil(t, err) - i := 0 - for { - _, ok, err := dbFrame.Next() - require.Nil(t, err) - if !ok { - break - } - i++ - } - require.Equal(t, 0, i) - }) -} - -func mockRowsToSqlRows(mockRows *sqlmock.Rows) *sql.Rows { - db, mock, _ := sqlmock.New() - mock.ExpectQuery("select").WillReturnRows(mockRows) - rows, _ := db.Query("select") - return rows -} diff --git a/programs/diagnostics/internal/platform/data/field.go b/programs/diagnostics/internal/platform/data/field.go deleted file mode 100644 index 5e80fc1f467..00000000000 --- a/programs/diagnostics/internal/platform/data/field.go +++ /dev/null @@ -1,8 +0,0 @@ -package data - -type Field struct { - // Name of the field - Name string - // A list of fields that must implement FieldType interface - Values []interface{} -} diff --git a/programs/diagnostics/internal/platform/data/file.go b/programs/diagnostics/internal/platform/data/file.go deleted file mode 100644 index 9760b4b6906..00000000000 --- a/programs/diagnostics/internal/platform/data/file.go +++ /dev/null @@ -1,444 +0,0 @@ -package data - -import ( - "bufio" - "encoding/xml" - "io/ioutil" - "os" - "path" - "path/filepath" - "regexp" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" -) - -type File interface { - Copy(destPath string, removeSensitive bool) error - FilePath() string -} - -type SimpleFile struct { - Path string -} - -// Copy supports removeSensitive for other file types but for a simple file this doesn't do anything -func (s SimpleFile) Copy(destPath string, removeSensitive bool) error { - // simple copy easiest - if err := utils.CopyFile(s.FilePath(), destPath); err != nil { - return errors.Wrapf(err, "unable to copy file %s", s.FilePath()) - } - return nil -} - -func (s SimpleFile) FilePath() string { - return s.Path -} - -func NewFileFrame(name string, filePaths []string) FileFrame { - i := 0 - files := make([]File, len(filePaths)) - for i, path := range filePaths { - files[i] = SimpleFile{ - Path: path, - } - } - return FileFrame{ - name: name, - i: &i, - files: files, - } -} - -type FileFrame struct { - name string - i *int - files []File -} - -func (f FileFrame) Next() ([]interface{}, bool, error) { - if len(f.files) == *(f.i) { - return nil, false, nil - } - file := f.files[*f.i] - *f.i++ - value := make([]interface{}, 1) - value[0] = file - return value, true, nil -} - -func (f FileFrame) Columns() []string { - return []string{"files"} -} - -func (f FileFrame) Name() string { - return f.name -} - -// DirectoryFileFrame represents a set of files under a directory -type DirectoryFileFrame struct { - FileFrame - Directory string -} - -func NewFileDirectoryFrame(directory string, exts []string) (DirectoryFileFrame, []error) { - filePaths, errs := utils.ListFilesInDirectory(directory, exts) - files := make([]File, len(filePaths)) - for i, path := range filePaths { - files[i] = SimpleFile{ - Path: path, - } - } - i := 0 - return DirectoryFileFrame{ - Directory: directory, - FileFrame: FileFrame{ - files: files, - i: &i, - }, - }, errs -} - -func (f DirectoryFileFrame) Next() ([]interface{}, bool, error) { - if len(f.files) == *(f.i) { - return nil, false, nil - } - file := f.files[*f.i] - *f.i++ - value := make([]interface{}, 1) - value[0] = file - return value, true, nil -} - -func (f DirectoryFileFrame) Columns() []string { - return []string{"files"} -} - -func (f DirectoryFileFrame) Name() string { - return f.Directory -} - -type ConfigFile interface { - File - FindLogPaths() ([]string, error) - FindIncludedConfig() (ConfigFile, error) - IsIncluded() bool -} - -type ConfigFileFrame struct { - i *int - Directory string - files []ConfigFile -} - -func (f ConfigFileFrame) Next() ([]interface{}, bool, error) { - if len(f.files) == *(f.i) { - return nil, false, nil - } - file := f.files[*f.i] - *f.i++ - value := make([]interface{}, 1) - value[0] = file - return value, true, nil -} - -func (f ConfigFileFrame) Name() string { - return f.Directory -} - -func NewConfigFileFrame(directory string) (ConfigFileFrame, []error) { - files, errs := utils.ListFilesInDirectory(directory, []string{"*.xml", "*.yaml", "*.yml"}) - // we can't predict the length because of include files - var configs []ConfigFile - - for _, path := range files { - var configFile ConfigFile - switch ext := filepath.Ext(path); ext { - case ".xml": - configFile = XmlConfigFile{ - Path: path, - Included: false, - } - case ".yml": - configFile = YamlConfigFile{ - Path: path, - Included: false, - } - case ".yaml": - configFile = YamlConfigFile{ - Path: path, - } - } - if configFile != nil { - configs = append(configs, configFile) - // add any included configs - iConf, err := configFile.FindIncludedConfig() - if err != nil { - errs = append(errs, err) - } else { - if iConf.FilePath() != "" { - configs = append(configs, iConf) - } - } - } - } - i := 0 - - return ConfigFileFrame{ - i: &i, - Directory: directory, - files: configs, - }, errs -} - -func (f ConfigFileFrame) Columns() []string { - return []string{"config"} -} - -func (f ConfigFileFrame) FindLogPaths() (logPaths []string, errors []error) { - for _, configFile := range f.files { - paths, err := configFile.FindLogPaths() - if err != nil { - errors = append(errors, err) - } else { - logPaths = append(logPaths, paths...) - } - } - return logPaths, errors -} - -type XmlConfigFile struct { - Path string - Included bool -} - -// these patterns will be used to remove sensitive content - matches of the pattern will be replaced with the key -var xmlSensitivePatterns = map[string]*regexp.Regexp{ - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), -} - -func (x XmlConfigFile) Copy(destPath string, removeSensitive bool) error { - if !removeSensitive { - // simple copy easiest - if err := utils.CopyFile(x.FilePath(), destPath); err != nil { - return errors.Wrapf(err, "unable to copy file %s", x.FilePath()) - } - return nil - } - return sensitiveFileCopy(x.FilePath(), destPath, xmlSensitivePatterns) -} - -func (x XmlConfigFile) FilePath() string { - return x.Path -} - -func (x XmlConfigFile) IsIncluded() bool { - return x.Included -} - -type XmlLoggerConfig struct { - XMLName xml.Name `xml:"logger"` - ErrorLog string `xml:"errorlog"` - Log string `xml:"log"` -} - -type YandexXMLConfig struct { - XMLName xml.Name `xml:"yandex"` - Clickhouse XmlLoggerConfig `xml:"logger"` - IncludeFrom string `xml:"include_from"` -} - -type XmlConfig struct { - XMLName xml.Name `xml:"clickhouse"` - Clickhouse XmlLoggerConfig `xml:"logger"` - IncludeFrom string `xml:"include_from"` -} - -func (x XmlConfigFile) UnmarshallConfig() (XmlConfig, error) { - inputFile, err := ioutil.ReadFile(x.Path) - - if err != nil { - return XmlConfig{}, err - } - var cConfig XmlConfig - err = xml.Unmarshal(inputFile, &cConfig) - if err == nil { - return XmlConfig{ - Clickhouse: cConfig.Clickhouse, - IncludeFrom: cConfig.IncludeFrom, - }, nil - } - // attempt to marshall as yandex file - var yConfig YandexXMLConfig - err = xml.Unmarshal(inputFile, &yConfig) - if err != nil { - return XmlConfig{}, err - } - return XmlConfig{ - Clickhouse: yConfig.Clickhouse, - IncludeFrom: yConfig.IncludeFrom, - }, nil -} - -func (x XmlConfigFile) FindLogPaths() ([]string, error) { - var paths []string - config, err := x.UnmarshallConfig() - if err != nil { - return nil, err - } - if config.Clickhouse.Log != "" { - paths = append(paths, config.Clickhouse.Log) - } - if config.Clickhouse.ErrorLog != "" { - paths = append(paths, config.Clickhouse.ErrorLog) - } - - return paths, nil -} - -func (x XmlConfigFile) FindIncludedConfig() (ConfigFile, error) { - if x.Included { - //can't recurse - return XmlConfigFile{}, nil - } - config, err := x.UnmarshallConfig() - if err != nil { - return XmlConfigFile{}, err - } - // we need to convert this - if config.IncludeFrom != "" { - if filepath.IsAbs(config.IncludeFrom) { - return XmlConfigFile{Path: config.IncludeFrom, Included: true}, nil - } - confDir := filepath.Dir(x.FilePath()) - return XmlConfigFile{Path: path.Join(confDir, config.IncludeFrom), Included: true}, nil - } - return XmlConfigFile{}, nil -} - -type YamlConfigFile struct { - Path string - Included bool -} - -var ymlSensitivePatterns = map[string]*regexp.Regexp{ - "password: 'Replaced'": regexp.MustCompile(`password:\s*.*$`), - "password_sha256_hex: 'Replaced'": regexp.MustCompile(`password_sha256_hex:\s*.*$`), - "access_key_id: 'Replaced'": regexp.MustCompile(`access_key_id:\s*.*$`), - "secret_access_key: 'Replaced'": regexp.MustCompile(`secret_access_key:\s*.*$`), - "secret: 'Replaced'": regexp.MustCompile(`secret:\s*.*$`), -} - -func (y YamlConfigFile) Copy(destPath string, removeSensitive bool) error { - if !removeSensitive { - // simple copy easiest - if err := utils.CopyFile(y.FilePath(), destPath); err != nil { - return errors.Wrapf(err, "unable to copy file %s", y.FilePath()) - } - return nil - } - return sensitiveFileCopy(y.FilePath(), destPath, ymlSensitivePatterns) -} - -func (y YamlConfigFile) FilePath() string { - return y.Path -} - -func (y YamlConfigFile) IsIncluded() bool { - return y.Included -} - -type YamlLoggerConfig struct { - Log string - ErrorLog string -} - -type YamlConfig struct { - Logger YamlLoggerConfig - Include_From string -} - -func (y YamlConfigFile) FindLogPaths() ([]string, error) { - var paths []string - inputFile, err := ioutil.ReadFile(y.Path) - if err != nil { - return nil, err - } - var config YamlConfig - err = yaml.Unmarshal(inputFile, &config) - if err != nil { - return nil, err - } - if config.Logger.Log != "" { - paths = append(paths, config.Logger.Log) - } - if config.Logger.ErrorLog != "" { - paths = append(paths, config.Logger.ErrorLog) - } - return paths, nil -} - -func (y YamlConfigFile) FindIncludedConfig() (ConfigFile, error) { - if y.Included { - //can't recurse - return YamlConfigFile{}, nil - } - inputFile, err := ioutil.ReadFile(y.Path) - if err != nil { - return YamlConfigFile{}, err - } - var config YamlConfig - err = yaml.Unmarshal(inputFile, &config) - if err != nil { - return YamlConfigFile{}, err - } - if config.Include_From != "" { - if filepath.IsAbs(config.Include_From) { - return YamlConfigFile{Path: config.Include_From, Included: true}, nil - } - confDir := filepath.Dir(y.FilePath()) - return YamlConfigFile{Path: path.Join(confDir, config.Include_From), Included: true}, nil - } - return YamlConfigFile{}, nil -} - -func sensitiveFileCopy(sourcePath string, destPath string, patterns map[string]*regexp.Regexp) error { - destDir := filepath.Dir(destPath) - if err := os.MkdirAll(destDir, os.ModePerm); err != nil { - return errors.Wrapf(err, "unable to create directory %s", destDir) - } - // currently, we don't unmarshall into a struct - we want to preserve structure and comments. Possibly could - // be handled but for simplicity we do a line parse for now - inputFile, err := os.Open(sourcePath) - - if err != nil { - return err - } - defer inputFile.Close() - outputFile, err := os.Create(destPath) - - if err != nil { - return err - } - defer outputFile.Close() - writer := bufio.NewWriter(outputFile) - scanner := bufio.NewScanner(inputFile) - - for scanner.Scan() { - line := scanner.Text() - for repl, pattern := range patterns { - line = pattern.ReplaceAllString(line, repl) - } - _, err = writer.WriteString(line + "\n") - if err != nil { - return err - } - } - writer.Flush() - return nil -} diff --git a/programs/diagnostics/internal/platform/data/file_test.go b/programs/diagnostics/internal/platform/data/file_test.go deleted file mode 100644 index 9e305b1a5da..00000000000 --- a/programs/diagnostics/internal/platform/data/file_test.go +++ /dev/null @@ -1,263 +0,0 @@ -package data_test - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestNextFileDirectoryFrame(t *testing.T) { - t.Run("can iterate file frame", func(t *testing.T) { - tempDir := t.TempDir() - files := make([]string, 5) - for i := 0; i < 5; i++ { - fileDir := path.Join(tempDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.txt", i)) - files[i] = filepath - _, err = os.Create(filepath) - require.Nil(t, err) - } - fileFrame, errs := data.NewFileDirectoryFrame(tempDir, []string{"*.txt"}) - require.Empty(t, errs) - i := 0 - for { - values, ok, err := fileFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - require.Equal(t, files[i], values[0].(data.SimpleFile).Path) - i += 1 - } - require.Equal(t, 5, i) - }) - - t.Run("can iterate file frame when empty", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - fileFrame, errs := data.NewFileDirectoryFrame(tempDir, []string{"*"}) - require.Empty(t, errs) - i := 0 - for { - _, ok, err := fileFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) -} - -func TestNewConfigFileFrame(t *testing.T) { - t.Run("can iterate config file frame", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - i := 0 - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - filePath := values[0].(data.XmlConfigFile).FilePath() - require.True(t, strings.Contains(filePath, ".xml")) - i += 1 - } - // 5 not 3 due to the includes - require.Equal(t, 5, i) - }) - - t.Run("can iterate file frame when empty", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - configFrame, errs := data.NewConfigFileFrame(tempDir) - require.Empty(t, errs) - i := 0 - for { - _, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) -} - -func TestConfigFileFrameCopy(t *testing.T) { - t.Run("can copy non-sensitive xml config files", func(t *testing.T) { - tmrDir := t.TempDir() - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Nil(t, err) - require.True(t, ok) - configFile := values[0].(data.XmlConfigFile) - newPath := path.Join(tmrDir, filepath.Base(configFile.FilePath())) - err = configFile.Copy(newPath, false) - require.FileExists(t, newPath) - sourceInfo, _ := os.Stat(configFile.FilePath()) - destInfo, _ := os.Stat(newPath) - require.Equal(t, sourceInfo.Size(), destInfo.Size()) - require.Nil(t, err) - } - }) - - t.Run("can copy sensitive xml config files", func(t *testing.T) { - tmrDir := t.TempDir() - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - i := 0 - var checkedFiles []string - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Nil(t, err) - require.True(t, ok) - configFile := values[0].(data.XmlConfigFile) - fileName := filepath.Base(configFile.FilePath()) - newPath := path.Join(tmrDir, fileName) - err = configFile.Copy(newPath, true) - require.FileExists(t, newPath) - require.Nil(t, err) - bytes, err := ioutil.ReadFile(newPath) - require.Nil(t, err) - s := string(bytes) - checkedFiles = append(checkedFiles, fileName) - if fileName == "users.xml" || fileName == "default-password.xml" || fileName == "user-include.xml" { - require.True(t, strings.Contains(s, "Replaced") || - strings.Contains(s, "Replaced")) - require.NotContains(t, s, "REPLACE_ME") - require.NotContains(t, s, "REPLACE_ME") - } else if fileName == "config.xml" { - require.True(t, strings.Contains(s, "Replaced")) - require.True(t, strings.Contains(s, "Replaced")) - require.True(t, strings.Contains(s, "Replaced")) - require.NotContains(t, s, "REPLACE_ME") - require.NotContains(t, s, "REPLACE_ME") - require.NotContains(t, s, "REPLACE_ME") - } - i++ - } - require.ElementsMatch(t, []string{"users.xml", "default-password.xml", "user-include.xml", "config.xml", "server-include.xml"}, checkedFiles) - require.Equal(t, 5, i) - }) - - t.Run("can copy sensitive yaml config files", func(t *testing.T) { - tmrDir := t.TempDir() - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yaml")) - require.Empty(t, errs) - i := 0 - var checkedFiles []string - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Nil(t, err) - require.True(t, ok) - configFile := values[0].(data.YamlConfigFile) - fileName := filepath.Base(configFile.FilePath()) - newPath := path.Join(tmrDir, fileName) - err = configFile.Copy(newPath, true) - require.FileExists(t, newPath) - require.Nil(t, err) - bytes, err := ioutil.ReadFile(newPath) - require.Nil(t, err) - s := string(bytes) - checkedFiles = append(checkedFiles, fileName) - if fileName == "users.yaml" || fileName == "default-password.yaml" || fileName == "user-include.yaml" { - require.True(t, strings.Contains(s, "password: 'Replaced'") || - strings.Contains(s, "password_sha256_hex: 'Replaced'")) - require.NotContains(t, s, "password: 'REPLACE_ME'") - require.NotContains(t, s, "password_sha256_hex: \"REPLACE_ME\"") - } else if fileName == "config.yaml" { - require.True(t, strings.Contains(s, "access_key_id: 'Replaced'")) - require.True(t, strings.Contains(s, "secret_access_key: 'Replaced'")) - require.True(t, strings.Contains(s, "secret: 'Replaced'")) - require.NotContains(t, s, "access_key_id: 'REPLACE_ME'") - require.NotContains(t, s, "secret_access_key: REPLACE_ME") - require.NotContains(t, s, "secret: REPLACE_ME") - } - i++ - } - require.ElementsMatch(t, []string{"users.yaml", "default-password.yaml", "user-include.yaml", "config.yaml", "server-include.yaml"}, checkedFiles) - require.Equal(t, 5, i) - }) -} - -func TestConfigFileFrameFindLogPaths(t *testing.T) { - t.Run("can find xml log paths", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.ElementsMatch(t, []string{"/var/log/clickhouse-server/clickhouse-server.log", - "/var/log/clickhouse-server/clickhouse-server.err.log"}, paths) - }) - - t.Run("can handle empty log paths", func(t *testing.T) { - configFrame, errs := data.NewConfigFileFrame(t.TempDir()) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.Empty(t, paths) - }) - - t.Run("can find yaml log paths", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yaml")) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.ElementsMatch(t, []string{"/var/log/clickhouse-server/clickhouse-server.log", - "/var/log/clickhouse-server/clickhouse-server.err.log"}, paths) - }) -} - -// test the legacy format for ClickHouse xml config files with a yandex root tag -func TestYandexConfigFile(t *testing.T) { - t.Run("can find xml log paths with yandex root", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yandex_xml")) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.ElementsMatch(t, []string{"/var/log/clickhouse-server/clickhouse-server.log", - "/var/log/clickhouse-server/clickhouse-server.err.log"}, paths) - }) -} diff --git a/programs/diagnostics/internal/platform/data/frame.go b/programs/diagnostics/internal/platform/data/frame.go deleted file mode 100644 index 65978430109..00000000000 --- a/programs/diagnostics/internal/platform/data/frame.go +++ /dev/null @@ -1,11 +0,0 @@ -package data - -type BaseFrame struct { - Name string -} - -type Frame interface { - Next() ([]interface{}, bool, error) - Columns() []string - Name() string -} diff --git a/programs/diagnostics/internal/platform/data/memory.go b/programs/diagnostics/internal/platform/data/memory.go deleted file mode 100644 index 25da25cf251..00000000000 --- a/programs/diagnostics/internal/platform/data/memory.go +++ /dev/null @@ -1,35 +0,0 @@ -package data - -type MemoryFrame struct { - i *int - ColumnNames []string - Rows [][]interface{} - name string -} - -func NewMemoryFrame(name string, columns []string, rows [][]interface{}) MemoryFrame { - i := 0 - return MemoryFrame{ - i: &i, - Rows: rows, - ColumnNames: columns, - name: name, - } -} - -func (f MemoryFrame) Next() ([]interface{}, bool, error) { - if f.Rows == nil || len(f.Rows) == *(f.i) { - return nil, false, nil - } - value := f.Rows[*f.i] - *f.i++ - return value, true, nil -} - -func (f MemoryFrame) Columns() []string { - return f.ColumnNames -} - -func (f MemoryFrame) Name() string { - return f.name -} diff --git a/programs/diagnostics/internal/platform/data/memory_test.go b/programs/diagnostics/internal/platform/data/memory_test.go deleted file mode 100644 index fcc02e37d32..00000000000 --- a/programs/diagnostics/internal/platform/data/memory_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package data_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestNextMemoryFrame(t *testing.T) { - t.Run("can iterate memory frame", func(t *testing.T) { - columns := []string{"Filesystem", "Size", "Used", "Avail", "Use%", "Mounted on"} - rows := [][]interface{}{ - {"sysfs", 0, 0, 0, 0, "/sys"}, - {"proc", 0, 0, 0, 0, "/proc"}, - {"udev", 33357840384, 0, 33357840384, 0, "/dev"}, - {"devpts", 0, 0, 0, 0, "/dev/pts"}, - {"tmpfs", 6682607616, 2228224, 6680379392, 1, "/run"}, - {"/dev/mapper/system-root", 1938213220352, 118136926208, 1721548947456, 7.000000000000001, "/"}, - } - memoryFrame := data.NewMemoryFrame("disks", columns, rows) - i := 0 - for { - values, ok, err := memoryFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.ElementsMatch(t, values, rows[i]) - require.Len(t, values, 6) - i += 1 - } - require.Equal(t, 6, i) - }) - - t.Run("can iterate memory frame when empty", func(t *testing.T) { - memoryFrame := data.NewMemoryFrame("test", []string{}, [][]interface{}{}) - i := 0 - for { - _, ok, err := memoryFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) - - t.Run("can iterate memory frame when empty", func(t *testing.T) { - memoryFrame := data.MemoryFrame{} - i := 0 - for { - _, ok, err := memoryFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) -} diff --git a/programs/diagnostics/internal/platform/data/misc.go b/programs/diagnostics/internal/platform/data/misc.go deleted file mode 100644 index a03213c4f46..00000000000 --- a/programs/diagnostics/internal/platform/data/misc.go +++ /dev/null @@ -1,27 +0,0 @@ -package data - -func NewHierarchicalFrame(name string, frame Frame, subFrames []HierarchicalFrame) HierarchicalFrame { - return HierarchicalFrame{ - name: name, - DataFrame: frame, - SubFrames: subFrames, - } -} - -type HierarchicalFrame struct { - name string - DataFrame Frame - SubFrames []HierarchicalFrame -} - -func (hf HierarchicalFrame) Name() string { - return hf.name -} - -func (hf HierarchicalFrame) Columns() []string { - return hf.DataFrame.Columns() -} - -func (hf HierarchicalFrame) Next() ([]interface{}, bool, error) { - return hf.DataFrame.Next() -} diff --git a/programs/diagnostics/internal/platform/database/native.go b/programs/diagnostics/internal/platform/database/native.go deleted file mode 100644 index 45b9af0349e..00000000000 --- a/programs/diagnostics/internal/platform/database/native.go +++ /dev/null @@ -1,95 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "net/url" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - _ "github.com/ClickHouse/clickhouse-go/v2" - "github.com/pkg/errors" -) - -type ClickhouseNativeClient struct { - host string - connection *sql.DB -} - -func NewNativeClient(host string, port uint16, username string, password string) (*ClickhouseNativeClient, error) { - // debug output ?debug=true - connection, err := sql.Open("clickhouse", fmt.Sprintf("clickhouse://%s:%s@%s:%d/", url.QueryEscape(username), url.QueryEscape(password), host, port)) - if err != nil { - return &ClickhouseNativeClient{}, err - } - if err := connection.Ping(); err != nil { - return &ClickhouseNativeClient{}, err - } - return &ClickhouseNativeClient{ - host: host, - connection: connection, - }, nil -} - -func (c *ClickhouseNativeClient) Ping() error { - return c.connection.Ping() -} - -func (c *ClickhouseNativeClient) ReadTable(databaseName string, tableName string, excludeColumns []string, orderBy data.OrderBy, limit int64) (data.Frame, error) { - exceptClause := "" - if len(excludeColumns) > 0 { - exceptClause = fmt.Sprintf("EXCEPT(%s) ", strings.Join(excludeColumns, ",")) - } - limitClause := "" - if limit >= 0 { - limitClause = fmt.Sprintf(" LIMIT %d", limit) - } - rows, err := c.connection.Query(fmt.Sprintf("SELECT * %sFROM %s.%s%s%s", exceptClause, databaseName, tableName, orderBy.String(), limitClause)) - if err != nil { - return data.DatabaseFrame{}, err - } - return data.NewDatabaseFrame(fmt.Sprintf("%s.%s", databaseName, tableName), rows) -} - -func (c *ClickhouseNativeClient) ReadTableNamesForDatabase(databaseName string) ([]string, error) { - rows, err := c.connection.Query(fmt.Sprintf("SHOW TABLES FROM %s", databaseName)) - if err != nil { - return nil, err - } - defer rows.Close() - var tableNames []string - var name string - for rows.Next() { - if err := rows.Scan(&name); err != nil { - return nil, err - } - tableNames = append(tableNames, name) - } - return tableNames, nil -} - -func (c *ClickhouseNativeClient) ExecuteStatement(id string, statement string) (data.Frame, error) { - rows, err := c.connection.Query(statement) - if err != nil { - return data.DatabaseFrame{}, err - } - return data.NewDatabaseFrame(id, rows) -} - -func (c *ClickhouseNativeClient) Version() (string, error) { - frame, err := c.ExecuteStatement("version", "SELECT version() as version") - if err != nil { - return "", err - } - values, ok, err := frame.Next() - if err != nil { - return "", err - } - if !ok { - return "", errors.New("unable to read ClickHouse version") - } - if len(values) != 1 { - return "", errors.New("unable to read ClickHouse version - no rows returned") - } - return values[0].(string), nil -} diff --git a/programs/diagnostics/internal/platform/database/native_test.go b/programs/diagnostics/internal/platform/database/native_test.go deleted file mode 100644 index 7028a4b4800..00000000000 --- a/programs/diagnostics/internal/platform/database/native_test.go +++ /dev/null @@ -1,289 +0,0 @@ -//go:build !no_docker - -package database_test - -import ( - "context" - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/docker/go-connections/nat" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -func createClickHouseContainer(t *testing.T, ctx context.Context) (testcontainers.Container, nat.Port) { - // create a ClickHouse container - cwd, err := os.Getwd() - if err != nil { - // can't test without current directory - panic(err) - } - - // for now, we test against a hardcoded database-server version but we should make this a property - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../../testdata/docker/admin.xml"), - }, - Target: "/etc/clickhouse-server/users.d/admin.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, _ := clickhouseContainer.MappedPort(ctx, "9000") - if err != nil { - // can't test without container's port - panic(err) - } - - t.Setenv("CLICKHOUSE_DB_PORT", p.Port()) - - return clickhouseContainer, p -} - -func getClient(t *testing.T, mappedPort int) *database.ClickhouseNativeClient { - clickhouseClient, err := database.NewNativeClient("localhost", uint16(mappedPort), "", "") - if err != nil { - t.Fatalf("unable to build client : %v", err) - } - return clickhouseClient -} - -func TestReadTableNamesForDatabase(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - t.Run("client can read tables for a database", func(t *testing.T) { - tables, err := clickhouseClient.ReadTableNamesForDatabase("system") - require.Nil(t, err) - require.GreaterOrEqual(t, len(tables), 70) - require.Contains(t, tables, "merge_tree_settings") - }) -} - -func TestReadTable(t *testing.T) { - t.Run("client can get all rows for system.disks table", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - // we read the table system.disks as this should contain only 1 row - frame, err := clickhouseClient.ReadTable("system", "disks", []string{}, data.OrderBy{}, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [9]string{"name", "path", "free_space", "total_space", "unreserved_space", "keep_free_space", "type", "is_encrypted", "cache_path"}) - i := 0 - for { - values, ok, err := frame.Next() - if i == 0 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, "default", values[0]) - require.Equal(t, "/var/lib/clickhouse/", values[1]) - require.Greater(t, values[2], uint64(0)) - require.Greater(t, values[3], uint64(0)) - require.Greater(t, values[4], uint64(0)) - require.Equal(t, values[5], uint64(0)) - require.Equal(t, "local", values[6]) - require.Equal(t, values[7], uint8(0)) - require.Equal(t, values[8], "") - } else { - require.False(t, ok) - break - } - i += 1 - } - }) - - t.Run("client can get all rows for system.databases table", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - // we read the table system.databases as this should be small and consistent on fresh db instances - frame, err := clickhouseClient.ReadTable("system", "databases", []string{}, data.OrderBy{}, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [6]string{"name", "engine", "data_path", "metadata_path", "uuid", "comment"}) - expectedRows := [4][3]string{{"INFORMATION_SCHEMA", "Memory", "/var/lib/clickhouse/"}, - {"default", "Atomic", "/var/lib/clickhouse/store/"}, - {"information_schema", "Memory", "/var/lib/clickhouse/"}, - {"system", "Atomic", "/var/lib/clickhouse/store/"}} - i := 0 - for { - values, ok, err := frame.Next() - - if i < 4 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - require.Equal(t, expectedRows[i][2], values[2]) - require.NotNil(t, values[3]) - require.NotNil(t, values[4]) - require.Equal(t, "", values[5]) - } else { - require.False(t, ok) - break - } - i += 1 - } - }) - - t.Run("client can get all rows for system.databases table with except", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - frame, err := clickhouseClient.ReadTable("system", "databases", []string{"data_path", "comment"}, data.OrderBy{}, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [4]string{"name", "engine", "metadata_path", "uuid"}) - }) - - t.Run("client can limit rows for system.databases", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - frame, err := clickhouseClient.ReadTable("system", "databases", []string{}, data.OrderBy{}, 1) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [6]string{"name", "engine", "data_path", "metadata_path", "uuid", "comment"}) - expectedRows := [1][3]string{{"INFORMATION_SCHEMA", "Memory", "/var/lib/clickhouse/"}} - i := 0 - for { - values, ok, err := frame.Next() - if i == 0 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - require.Equal(t, expectedRows[i][2], values[2]) - require.NotNil(t, values[3]) - require.NotNil(t, values[4]) - require.Equal(t, "", values[5]) - } else { - require.False(t, ok) - break - } - i += 1 - } - }) - - t.Run("client can order rows for system.databases", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - frame, err := clickhouseClient.ReadTable("system", "databases", []string{}, data.OrderBy{ - Column: "engine", - Order: data.Asc, - }, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [6]string{"name", "engine", "data_path", "metadata_path", "uuid", "comment"}) - expectedRows := [4][3]string{ - {"default", "Atomic", "/var/lib/clickhouse/store/"}, - {"system", "Atomic", "/var/lib/clickhouse/store/"}, - {"INFORMATION_SCHEMA", "Memory", "/var/lib/clickhouse/"}, - {"information_schema", "Memory", "/var/lib/clickhouse/"}, - } - i := 0 - for { - values, ok, err := frame.Next() - - if i < 4 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - require.Equal(t, expectedRows[i][2], values[2]) - require.NotNil(t, values[3]) - require.NotNil(t, values[4]) - require.Equal(t, "", values[5]) - } else { - require.False(t, ok) - break - } - i += 1 - } - }) -} - -func TestExecuteStatement(t *testing.T) { - t.Run("client can execute any statement", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - statement := "SELECT path, count(*) as count FROM system.disks GROUP BY path;" - frame, err := clickhouseClient.ExecuteStatement("engines", statement) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [2]string{"path", "count"}) - expectedRows := [1][2]interface{}{ - {"/var/lib/clickhouse/", uint64(1)}, - } - i := 0 - for { - values, ok, err := frame.Next() - if !ok { - require.Nil(t, err) - break - } - require.Nil(t, err) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - i++ - } - fmt.Println(i) - }) -} - -func TestVersion(t *testing.T) { - t.Run("client can read version", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - version, err := clickhouseClient.Version() - require.Nil(t, err) - require.NotEmpty(t, version) - }) -} diff --git a/programs/diagnostics/internal/platform/manager.go b/programs/diagnostics/internal/platform/manager.go deleted file mode 100644 index b4435b62ea2..00000000000 --- a/programs/diagnostics/internal/platform/manager.go +++ /dev/null @@ -1,49 +0,0 @@ -package platform - -import ( - "errors" - "sync" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" -) - -var once sync.Once -var dbInit sync.Once - -// manages all resources that collectors and outputs may wish to ensure inc. db connections - -type DBClient interface { - ReadTableNamesForDatabase(databaseName string) ([]string, error) - ReadTable(databaseName string, tableName string, excludeColumns []string, orderBy data.OrderBy, limit int64) (data.Frame, error) - ExecuteStatement(id string, statement string) (data.Frame, error) - Version() (string, error) -} - -var manager *ResourceManager - -type ResourceManager struct { - DbClient DBClient -} - -func GetResourceManager() *ResourceManager { - once.Do(func() { - manager = &ResourceManager{} - }) - return manager -} - -func (m *ResourceManager) Connect(host string, port uint16, username string, password string) error { - var err error - var clientInstance DBClient - init := false - dbInit.Do(func() { - clientInstance, err = database.NewNativeClient(host, port, username, password) - manager.DbClient = clientInstance - init = true - }) - if !init { - return errors.New("connect can only be called once") - } - return err -} diff --git a/programs/diagnostics/internal/platform/manager_test.go b/programs/diagnostics/internal/platform/manager_test.go deleted file mode 100644 index e6c50c6e505..00000000000 --- a/programs/diagnostics/internal/platform/manager_test.go +++ /dev/null @@ -1,100 +0,0 @@ -//go:build !no_docker - -package platform_test - -import ( - "context" - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/docker/go-connections/nat" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -// create a ClickHouse container -func createClickHouseContainer(t *testing.T, ctx context.Context) (testcontainers.Container, nat.Port) { - cwd, err := os.Getwd() - if err != nil { - fmt.Println("unable to read current directory", err) - os.Exit(1) - } - // for now, we test against a hardcoded database-server version but we should make this a property - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../testdata/docker/admin.xml"), - }, - Target: "/etc/clickhouse-server/users.d/admin.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, err := clickhouseContainer.MappedPort(ctx, "9000") - if err != nil { - // can't test without a port - panic(err) - } - - return clickhouseContainer, p -} - -func TestConnect(t *testing.T) { - t.Run("can only connect once", func(t *testing.T) { - ctx := context.Background() - - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - t.Setenv("CLICKHOUSE_DB_PORT", mappedPort.Port()) - - port := mappedPort.Int() - - // get before connection - manager := platform.GetResourceManager() - require.Nil(t, manager.DbClient) - // init connection - err := manager.Connect("localhost", uint16(port), "", "") - require.Nil(t, err) - require.NotNil(t, manager.DbClient) - // try and re-fetch connection - err = manager.Connect("localhost", uint16(port), "", "") - require.NotNil(t, err) - require.Equal(t, "connect can only be called once", err.Error()) - }) - -} - -func TestGetResourceManager(t *testing.T) { - t.Run("get resource manager", func(t *testing.T) { - manager := platform.GetResourceManager() - require.NotNil(t, manager) - manager2 := platform.GetResourceManager() - require.NotNil(t, manager2) - require.Equal(t, &manager, &manager2) - }) - -} diff --git a/programs/diagnostics/internal/platform/test/data.go b/programs/diagnostics/internal/platform/test/data.go deleted file mode 100644 index 7710e9a69a1..00000000000 --- a/programs/diagnostics/internal/platform/test/data.go +++ /dev/null @@ -1,166 +0,0 @@ -package test - -import ( - "fmt" - "sort" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" -) - -type fakeClickhouseClient struct { - tables map[string][]string - QueryResponses map[string]*FakeDataFrame -} - -func NewFakeClickhouseClient(tables map[string][]string) fakeClickhouseClient { - queryResponses := make(map[string]*FakeDataFrame) - return fakeClickhouseClient{ - tables: tables, - QueryResponses: queryResponses, - } -} - -func (f fakeClickhouseClient) ReadTableNamesForDatabase(databaseName string) ([]string, error) { - if _, ok := f.tables[databaseName]; ok { - return f.tables[databaseName], nil - } - return nil, fmt.Errorf("database %s does not exist", databaseName) -} - -func (f fakeClickhouseClient) ReadTable(databaseName string, tableName string, excludeColumns []string, orderBy data.OrderBy, limit int64) (data.Frame, error) { - - exceptClause := "" - if len(excludeColumns) > 0 { - exceptClause = fmt.Sprintf("EXCEPT(%s) ", strings.Join(excludeColumns, ",")) - } - limitClause := "" - if limit >= 0 { - limitClause = fmt.Sprintf(" LIMIT %d", limit) - } - query := fmt.Sprintf("SELECT * %sFROM %s.%s%s%s", exceptClause, databaseName, tableName, orderBy.String(), limitClause) - frame, error := f.ExecuteStatement(fmt.Sprintf("read_table_%s.%s", databaseName, tableName), query) - if error != nil { - return frame, error - } - fFrame := *(frame.(*FakeDataFrame)) - fFrame = fFrame.FilterColumns(excludeColumns) - fFrame = fFrame.Order(orderBy) - fFrame = fFrame.Limit(limit) - return fFrame, nil -} - -func (f fakeClickhouseClient) ExecuteStatement(id string, statement string) (data.Frame, error) { - if frame, ok := f.QueryResponses[statement]; ok { - return frame, nil - } - return FakeDataFrame{}, errors.New(fmt.Sprintf("No recorded response for %s", statement)) -} - -func (f fakeClickhouseClient) Version() (string, error) { - return "21.12.3", nil -} - -func (f fakeClickhouseClient) Reset() { - for key, frame := range f.QueryResponses { - frame.Reset() - f.QueryResponses[key] = frame - } -} - -type FakeDataFrame struct { - i *int - Rows [][]interface{} - ColumnNames []string - name string -} - -func NewFakeDataFrame(name string, columns []string, rows [][]interface{}) FakeDataFrame { - i := 0 - return FakeDataFrame{ - i: &i, - Rows: rows, - ColumnNames: columns, - name: name, - } -} - -func (f FakeDataFrame) Next() ([]interface{}, bool, error) { - if len(f.Rows) == *(f.i) { - return nil, false, nil - } - value := f.Rows[*f.i] - *f.i++ - return value, true, nil -} - -func (f FakeDataFrame) Columns() []string { - return f.ColumnNames -} - -func (f FakeDataFrame) Name() string { - return f.name -} - -func (f *FakeDataFrame) Reset() { - i := 0 - f.i = &i -} - -func (f FakeDataFrame) FilterColumns(excludeColumns []string) FakeDataFrame { - // get columns we can remove - rColumns := utils.Intersection(f.ColumnNames, excludeColumns) - rIndexes := make([]int, len(rColumns)) - // find the indexes of the columns to remove - for i, column := range rColumns { - rIndexes[i] = utils.IndexOf(f.ColumnNames, column) - } - newRows := make([][]interface{}, len(f.Rows)) - for r, row := range f.Rows { - newRow := row - for i, index := range rIndexes { - newRow = utils.Remove(newRow, index-i) - } - newRows[r] = newRow - } - f.Rows = newRows - f.ColumnNames = utils.Distinct(f.ColumnNames, excludeColumns) - return f -} - -func (f FakeDataFrame) Limit(rowLimit int64) FakeDataFrame { - if rowLimit >= 0 { - if int64(len(f.Rows)) > rowLimit { - f.Rows = f.Rows[:rowLimit] - } - } - return f -} - -func (f FakeDataFrame) Order(orderBy data.OrderBy) FakeDataFrame { - if orderBy.Column == "" { - return f - } - cIndex := utils.IndexOf(f.ColumnNames, orderBy.Column) - sort.Slice(f.Rows, func(i, j int) bool { - left := f.Rows[i][cIndex] - right := f.Rows[j][cIndex] - if iLeft, ok := left.(int); ok { - if orderBy.Order == data.Asc { - return iLeft < right.(int) - } - return iLeft > right.(int) - } else { - // we aren't a full db - revert to string order - sLeft := left.(string) - sRight := right.(string) - if orderBy.Order == data.Asc { - return sLeft < sRight - } - return sLeft > sRight - } - }) - return f -} diff --git a/programs/diagnostics/internal/platform/test/env.go b/programs/diagnostics/internal/platform/test/env.go deleted file mode 100644 index 36b03772ab0..00000000000 --- a/programs/diagnostics/internal/platform/test/env.go +++ /dev/null @@ -1,16 +0,0 @@ -package test - -import "os" - -const defaultClickHouseVersion = "latest" - -func GetClickHouseTestVersion() string { - return GetEnv("CLICKHOUSE_VERSION", defaultClickHouseVersion) -} - -func GetEnv(key, fallback string) string { - if value, ok := os.LookupEnv(key); ok { - return value - } - return fallback -} diff --git a/programs/diagnostics/internal/platform/utils/file.go b/programs/diagnostics/internal/platform/utils/file.go deleted file mode 100644 index 71af4b32658..00000000000 --- a/programs/diagnostics/internal/platform/utils/file.go +++ /dev/null @@ -1,95 +0,0 @@ -package utils - -import ( - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -func FileExists(name string) (bool, error) { - f, err := os.Stat(name) - if err == nil { - if !f.IsDir() { - return true, nil - } - return false, fmt.Errorf("%s is a directory", name) - } - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - return false, err -} - -func DirExists(name string) (bool, error) { - f, err := os.Stat(name) - if err == nil { - if f.IsDir() { - return true, nil - } - return false, fmt.Errorf("%s is a file", name) - } - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - return false, err -} - -func CopyFile(sourceFilename string, destFilename string) error { - exists, err := FileExists(sourceFilename) - if err != nil { - return err - } - if !exists { - return fmt.Errorf("%s does not exist", sourceFilename) - } - source, err := os.Open(sourceFilename) - if err != nil { - return err - } - defer source.Close() - destDir := filepath.Dir(destFilename) - if err := os.MkdirAll(destDir, os.ModePerm); err != nil { - return errors.Wrapf(err, "unable to create directory %s", destDir) - } - - destination, err := os.Create(destFilename) - if err != nil { - return err - } - defer destination.Close() - _, err = io.Copy(destination, source) - return err -} - -// patterns passed are an OR - any can be satisfied and the file will be listed - -func ListFilesInDirectory(directory string, patterns []string) ([]string, []error) { - var files []string - exists, err := DirExists(directory) - if err != nil { - return files, []error{err} - } - if !exists { - return files, []error{fmt.Errorf("directory %s does not exist", directory)} - } - var pathErrors []error - _ = filepath.Walk(directory, func(path string, info fs.FileInfo, err error) error { - if err != nil { - pathErrors = append(pathErrors, err) - } else if !info.IsDir() { - for _, pattern := range patterns { - if matched, err := filepath.Match(pattern, filepath.Base(path)); err != nil { - pathErrors = append(pathErrors, err) - } else if matched { - files = append(files, path) - } - } - } - return nil - }) - return files, pathErrors -} diff --git a/programs/diagnostics/internal/platform/utils/file_test.go b/programs/diagnostics/internal/platform/utils/file_test.go deleted file mode 100644 index 8d0430090c9..00000000000 --- a/programs/diagnostics/internal/platform/utils/file_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package utils_test - -import ( - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" -) - -func TestFileExists(t *testing.T) { - t.Run("returns true for file", func(t *testing.T) { - tempDir := t.TempDir() - filepath := path.Join(tempDir, "random.txt") - _, err := os.Create(filepath) - require.Nil(t, err) - exists, err := utils.FileExists(filepath) - require.True(t, exists) - require.Nil(t, err) - }) - - t.Run("doesn't return true for not existence file", func(t *testing.T) { - tempDir := t.TempDir() - file := path.Join(tempDir, "random.txt") - exists, err := utils.FileExists(file) - require.False(t, exists) - require.Nil(t, err) - }) - - t.Run("doesn't return true for directory", func(t *testing.T) { - tempDir := t.TempDir() - exists, err := utils.FileExists(tempDir) - require.False(t, exists) - require.NotNil(t, err) - require.Equal(t, fmt.Sprintf("%s is a directory", tempDir), err.Error()) - }) -} - -func TestDirExists(t *testing.T) { - t.Run("doesn't return true for file", func(t *testing.T) { - tempDir := t.TempDir() - filepath := path.Join(tempDir, "random.txt") - _, err := os.Create(filepath) - require.Nil(t, err) - exists, err := utils.DirExists(filepath) - require.False(t, exists) - require.NotNil(t, err) - require.Equal(t, fmt.Sprintf("%s is a file", filepath), err.Error()) - }) - - t.Run("returns true for directory", func(t *testing.T) { - tempDir := t.TempDir() - exists, err := utils.DirExists(tempDir) - require.True(t, exists) - require.Nil(t, err) - }) - - t.Run("doesn't return true random directory", func(t *testing.T) { - exists, err := utils.FileExists(fmt.Sprintf("%d", utils.MakeTimestamp())) - require.False(t, exists) - require.Nil(t, err) - }) -} - -func TestCopyFile(t *testing.T) { - t.Run("can copy file", func(t *testing.T) { - tempDir := t.TempDir() - sourcePath := path.Join(tempDir, "random.txt") - _, err := os.Create(sourcePath) - require.Nil(t, err) - destPath := path.Join(tempDir, "random-2.txt") - err = utils.CopyFile(sourcePath, destPath) - require.Nil(t, err) - }) - - t.Run("can copy nested file", func(t *testing.T) { - tempDir := t.TempDir() - sourcePath := path.Join(tempDir, "random.txt") - _, err := os.Create(sourcePath) - require.Nil(t, err) - destPath := path.Join(tempDir, "sub_dir", "random-2.txt") - err = utils.CopyFile(sourcePath, destPath) - require.Nil(t, err) - }) - - t.Run("fails when file does not exist", func(t *testing.T) { - tempDir := t.TempDir() - sourcePath := path.Join(tempDir, "random.txt") - destPath := path.Join(tempDir, "random-2.txt") - err := utils.CopyFile(sourcePath, destPath) - require.NotNil(t, err) - require.Equal(t, fmt.Sprintf("%s does not exist", sourcePath), err.Error()) - }) -} - -func TestListFilesInDirectory(t *testing.T) { - tempDir := t.TempDir() - files := make([]string, 5) - for i := 0; i < 5; i++ { - fileDir := path.Join(tempDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - ext := ".txt" - if i%2 == 0 { - ext = ".csv" - } - filepath := path.Join(fileDir, fmt.Sprintf("random-%d%s", i, ext)) - files[i] = filepath - _, err = os.Create(filepath) - require.Nil(t, err) - } - - t.Run("can list all files", func(t *testing.T) { - mFiles, errs := utils.ListFilesInDirectory(tempDir, []string{"*"}) - require.Len(t, mFiles, 5) - require.Empty(t, errs) - }) - - t.Run("can list by extension", func(t *testing.T) { - mFiles, errs := utils.ListFilesInDirectory(tempDir, []string{"*.csv"}) - require.Len(t, mFiles, 3) - require.Empty(t, errs) - require.ElementsMatch(t, []string{files[0], files[2], files[4]}, mFiles) - }) - - t.Run("can list on multiple extensions files", func(t *testing.T) { - mFiles, errs := utils.ListFilesInDirectory(tempDir, []string{"*.csv", "*.txt"}) - require.Len(t, mFiles, 5) - require.Empty(t, errs) - }) - -} diff --git a/programs/diagnostics/internal/platform/utils/process.go b/programs/diagnostics/internal/platform/utils/process.go deleted file mode 100644 index 7b27c215eea..00000000000 --- a/programs/diagnostics/internal/platform/utils/process.go +++ /dev/null @@ -1,49 +0,0 @@ -package utils - -import ( - "github.com/elastic/gosigar" - "strings" -) - -func FindClickHouseProcesses() ([]gosigar.ProcArgs, error) { - pids := gosigar.ProcList{} - err := pids.Get() - if err != nil { - return nil, err - } - var clickhousePs []gosigar.ProcArgs - for _, pid := range pids.List { - args := gosigar.ProcArgs{} - if err := args.Get(pid); err != nil { - continue - } - if len(args.List) > 0 { - if strings.Contains(args.List[0], "clickhouse-server") { - clickhousePs = append(clickhousePs, args) - } - } - } - return clickhousePs, nil -} - -func FindConfigsFromClickHouseProcesses() ([]string, error) { - clickhouseProcesses, err := FindClickHouseProcesses() - if err != nil { - return nil, err - } - var configs []string - if len(clickhouseProcesses) > 0 { - // we have candidate matches - for _, ps := range clickhouseProcesses { - for _, arg := range ps.List { - if strings.Contains(arg, "--config") { - configFile := strings.ReplaceAll(arg, "--config-file=", "") - // containers receive config with --config - configFile = strings.ReplaceAll(configFile, "--config=", "") - configs = append(configs, configFile) - } - } - } - } - return configs, err -} diff --git a/programs/diagnostics/internal/platform/utils/process_test.go b/programs/diagnostics/internal/platform/utils/process_test.go deleted file mode 100644 index 9baaa559752..00000000000 --- a/programs/diagnostics/internal/platform/utils/process_test.go +++ /dev/null @@ -1,97 +0,0 @@ -//go:build !no_docker - -package utils_test - -import ( - "context" - "fmt" - "io" - "os" - "path" - "strings" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -func getProcessesInContainer(t *testing.T, container testcontainers.Container) ([]string, error) { - result, reader, err := container.Exec(context.Background(), []string{"ps", "-aux"}) - if err != nil { - return nil, err - } - require.Zero(t, result) - require.NotNil(t, reader) - - b, err := io.ReadAll(reader) - if err != nil { - return nil, err - } - require.NotNil(t, b) - - lines := strings.Split(string(b), "\n") - - // discard PS header - return lines[1:], nil -} - -func TestFindClickHouseProcessesAndConfigs(t *testing.T) { - - t.Run("can find ClickHouse processes and configs", func(t *testing.T) { - // create a ClickHouse container - ctx := context.Background() - cwd, err := os.Getwd() - if err != nil { - fmt.Println("unable to read current directory", err) - os.Exit(1) - } - - // run a ClickHouse container that guarantees that it runs only for the duration of the test - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, _ := clickhouseContainer.MappedPort(ctx, "9000") - - t.Setenv("CLICKHOUSE_DB_PORT", p.Port()) - - defer clickhouseContainer.Terminate(ctx) //nolint - - lines, err := getProcessesInContainer(t, clickhouseContainer) - require.Nil(t, err) - require.NotEmpty(t, lines) - - for _, line := range lines { - parts := strings.Fields(line) - if len(parts) < 11 { - continue - } - if !strings.Contains(parts[10], "clickhouse-server") { - continue - } - - require.Equal(t, "/usr/bin/clickhouse-server", parts[10]) - require.Equal(t, "--config-file=/etc/clickhouse-server/config.xml", parts[11]) - } - }) -} diff --git a/programs/diagnostics/internal/platform/utils/slices.go b/programs/diagnostics/internal/platform/utils/slices.go deleted file mode 100644 index cf5a5f97ce8..00000000000 --- a/programs/diagnostics/internal/platform/utils/slices.go +++ /dev/null @@ -1,68 +0,0 @@ -package utils - -// Intersection of elements in s1 and s2 -func Intersection(s1, s2 []string) (inter []string) { - hash := make(map[string]bool) - for _, e := range s1 { - hash[e] = false - } - for _, e := range s2 { - // If elements present in the hashmap then append intersection list. - if val, ok := hash[e]; ok { - if !val { - // only add once - inter = append(inter, e) - hash[e] = true - } - } - } - return inter -} - -// Distinct returns elements in s1, not in s2 -func Distinct(s1, s2 []string) (distinct []string) { - hash := make(map[string]bool) - for _, e := range s2 { - hash[e] = true - } - for _, e := range s1 { - if _, ok := hash[e]; !ok { - distinct = append(distinct, e) - } - } - return distinct -} - -// Unique func Unique(s1 []string) (unique []string) returns unique elements in s1 -func Unique(s1 []string) (unique []string) { - hash := make(map[string]bool) - for _, e := range s1 { - if _, ok := hash[e]; !ok { - unique = append(unique, e) - } - hash[e] = true - } - return unique -} - -func Contains(s []string, e string) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} - -func IndexOf(s []string, e string) int { - for i, a := range s { - if a == e { - return i - } - } - return -1 -} - -func Remove(slice []interface{}, s int) []interface{} { - return append(slice[:s], slice[s+1:]...) -} diff --git a/programs/diagnostics/internal/platform/utils/slices_test.go b/programs/diagnostics/internal/platform/utils/slices_test.go deleted file mode 100644 index ea5c1c81dcc..00000000000 --- a/programs/diagnostics/internal/platform/utils/slices_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package utils_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" -) - -func TestIntersection(t *testing.T) { - t.Run("can perform intersection", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - setB := []string{"A", "B", "F", "C", "G"} - setC := utils.Intersection(setA, setB) - require.Len(t, setC, 2) - require.ElementsMatch(t, []string{"A", "C"}, setC) - }) -} - -func TestDistinct(t *testing.T) { - t.Run("can perform distinct", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - setB := []string{"A", "B", "F", "C", "G"} - setC := utils.Distinct(setA, setB) - require.Len(t, setC, 3) - require.ElementsMatch(t, []string{"b", "D", "E"}, setC) - }) - - t.Run("can perform distinct on empty", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - var setB []string - setC := utils.Distinct(setA, setB) - require.Len(t, setC, 5) - require.ElementsMatch(t, []string{"A", "b", "C", "D", "E"}, setC) - }) -} - -func TestContains(t *testing.T) { - t.Run("can perform contains", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - require.True(t, utils.Contains(setA, "A")) - require.True(t, utils.Contains(setA, "b")) - require.True(t, utils.Contains(setA, "C")) - require.True(t, utils.Contains(setA, "D")) - require.True(t, utils.Contains(setA, "E")) - require.False(t, utils.Contains(setA, "B")) - }) -} - -func TestUnique(t *testing.T) { - - t.Run("can perform unique", func(t *testing.T) { - setA := []string{"A", "b", "D", "D", "E", "E", "A"} - setC := utils.Unique(setA) - require.Len(t, setC, 4) - require.ElementsMatch(t, []string{"A", "b", "D", "E"}, setC) - }) - - t.Run("can perform unique on empty", func(t *testing.T) { - var setA []string - setC := utils.Unique(setA) - require.Len(t, setC, 0) - }) -} diff --git a/programs/diagnostics/internal/platform/utils/time.go b/programs/diagnostics/internal/platform/utils/time.go deleted file mode 100644 index 622e92b873a..00000000000 --- a/programs/diagnostics/internal/platform/utils/time.go +++ /dev/null @@ -1,7 +0,0 @@ -package utils - -import "time" - -func MakeTimestamp() int64 { - return time.Now().UnixNano() / int64(time.Millisecond) -} diff --git a/programs/diagnostics/internal/runner.go b/programs/diagnostics/internal/runner.go deleted file mode 100644 index 9386a1d178b..00000000000 --- a/programs/diagnostics/internal/runner.go +++ /dev/null @@ -1,115 +0,0 @@ -package internal - -import ( - c "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - o "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -type runConfiguration struct { - id string - host string - port uint16 - username string - password string - output string - collectors []string - collectorConfigs map[string]config.Configuration - outputConfig config.Configuration -} - -func NewRunConfiguration(id string, host string, port uint16, username string, password string, output string, outputConfig config.Configuration, - collectors []string, collectorConfigs map[string]config.Configuration) *runConfiguration { - config := runConfiguration{ - id: id, - host: host, - port: port, - username: username, - password: password, - collectors: collectors, - output: output, - collectorConfigs: collectorConfigs, - outputConfig: outputConfig, - } - return &config -} - -func Capture(config *runConfiguration) { - bundles, err := collect(config) - if err != nil { - log.Fatal().Err(err).Msg("unable to perform collection") - } - log.Info().Msgf("collectors initialized") - if err = output(config, bundles); err != nil { - log.Fatal().Err(err).Msg("unable to create output") - } - log.Info().Msgf("bundle export complete") -} - -func collect(config *runConfiguration) (map[string]*data.DiagnosticBundle, error) { - resourceManager := platform.GetResourceManager() - err := resourceManager.Connect(config.host, config.port, config.username, config.password) - if err != nil { - // if we can't connect this is fatal - log.Fatal().Err(err).Msg("Unable to connect to database") - } - //grab the required connectors - we pass what we can - bundles := make(map[string]*data.DiagnosticBundle) - log.Info().Msgf("connection established") - //these store our collection errors and will be output in the bundle - var collectorErrors [][]interface{} - for _, collectorName := range config.collectors { - collectorConfig := config.collectorConfigs[collectorName] - log.Info().Msgf("initializing %s collector", collectorName) - collector, err := c.GetCollectorByName(collectorName) - if err != nil { - log.Error().Err(err).Msgf("Unable to fetch collector %s", collectorName) - collectorErrors = append(collectorErrors, []interface{}{err.Error()}) - continue - } - bundle, err := collector.Collect(collectorConfig) - if err != nil { - log.Error().Err(err).Msgf("Error in collector %s", collectorName) - collectorErrors = append(collectorErrors, []interface{}{err.Error()}) - // this indicates a fatal error in the collector - continue - } - for _, fError := range bundle.Errors.Errors { - err = errors.Wrapf(fError, "Failure to collect frame in collector %s", collectorName) - collectorErrors = append(collectorErrors, []interface{}{err.Error()}) - log.Warn().Msg(err.Error()) - } - bundles[collectorName] = bundle - } - bundles["diag_trace"] = buildTraceBundle(collectorErrors) - return bundles, nil -} - -func output(config *runConfiguration, bundles map[string]*data.DiagnosticBundle) error { - log.Info().Msgf("attempting to export bundle using %s output...", config.output) - output, err := o.GetOutputByName(config.output) - if err != nil { - return err - } - frameErrors, err := output.Write(config.id, bundles, config.outputConfig) - // we report over failing hard on frame errors - up to the output to determine what is fatal via error - for _, fError := range frameErrors.Errors { - log.Warn().Msgf("failure to write frame in output %s - %s", config.output, fError) - } - return err -} - -func buildTraceBundle(collectorErrors [][]interface{}) *data.DiagnosticBundle { - errorBundle := data.DiagnosticBundle{ - Frames: map[string]data.Frame{ - "errors": data.NewMemoryFrame("errors", []string{"errors"}, collectorErrors), - }, - Errors: data.FrameErrors{}, - } - // add any other metrics from collection - return &errorBundle -} diff --git a/programs/diagnostics/internal/runner_test.go b/programs/diagnostics/internal/runner_test.go deleted file mode 100644 index 2369f8b3007..00000000000 --- a/programs/diagnostics/internal/runner_test.go +++ /dev/null @@ -1,130 +0,0 @@ -//go:build !no_docker - -package internal_test - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -// Execute a full default capture, with simple output, and check if a bundle is produced and it's not empty -func TestCapture(t *testing.T) { - // create a ClickHouse container - ctx := context.Background() - cwd, err := os.Getwd() - - if err != nil { - // can't test without container - panic(err) - } - // for now, we test against a hardcoded database-server version but we should make this a property - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../testdata/docker/admin.xml"), - }, - Target: "/etc/clickhouse-server/users.d/admin.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, _ := clickhouseContainer.MappedPort(ctx, "9000") - - t.Setenv("CLICKHOUSE_DB_PORT", p.Port()) - defer clickhouseContainer.Terminate(ctx) //nolint - - tmrDir := t.TempDir() - port := p.Int() - - // test a simple output exists - _, err = outputs.GetOutputByName("simple") - require.Nil(t, err) - // this relies on the simple out not changing its params - test will likely fail if so - outputConfig := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: tmrDir, - Param: config.NewParam("directory", "Directory in which to create dump. Defaults to the current directory.", false), - }, - config.StringOptions{ - Value: "csv", - Options: []string{"csv"}, - Param: config.NewParam("format", "Format of exported files", false), - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Don't compress output to an archive", false), - }, - }, - } - // test default collectors - collectorNames := collectors.GetCollectorNames(true) - // grab all configs - only default will be used because of collectorNames - collectorConfigs, err := collectors.BuildConfigurationOptions() - require.Nil(t, err) - conf := internal.NewRunConfiguration("random", "localhost", uint16(port), "", "", "simple", outputConfig, collectorNames, collectorConfigs) - internal.Capture(conf) - outputDir := path.Join(tmrDir, "random") - _, err = os.Stat(outputDir) - require.Nil(t, err) - require.True(t, !os.IsNotExist(err)) - files, err := ioutil.ReadDir(outputDir) - require.Nil(t, err) - require.Len(t, files, 1) - outputDir = path.Join(outputDir, files[0].Name()) - // check we have a folder per collector i.e. collectorNames + diag_trace - files, err = ioutil.ReadDir(outputDir) - require.Nil(t, err) - require.Len(t, files, len(collectorNames)+1) - expectedFolders := append(collectorNames, "diag_trace") - for _, file := range files { - require.True(t, file.IsDir()) - utils.Contains(expectedFolders, file.Name()) - } - // we don't test the specific collector outputs but make sure something was written to system - systemFolder := path.Join(outputDir, "system") - files, err = ioutil.ReadDir(systemFolder) - require.Nil(t, err) - require.Greater(t, len(files), 0) - // test diag_trace - diagFolder := path.Join(outputDir, "diag_trace") - files, err = ioutil.ReadDir(diagFolder) - require.Nil(t, err) - require.Equal(t, 1, len(files)) - require.FileExists(t, path.Join(diagFolder, "errors.csv")) -} diff --git a/programs/diagnostics/testdata/configs/include/xml/server-include.xml b/programs/diagnostics/testdata/configs/include/xml/server-include.xml deleted file mode 100644 index 30e6587c935..00000000000 --- a/programs/diagnostics/testdata/configs/include/xml/server-include.xml +++ /dev/null @@ -1,8 +0,0 @@ - - 5000000 - - - - - 9008 - \ No newline at end of file diff --git a/programs/diagnostics/testdata/configs/include/xml/user-include.xml b/programs/diagnostics/testdata/configs/include/xml/user-include.xml deleted file mode 100644 index b12b34a56bb..00000000000 --- a/programs/diagnostics/testdata/configs/include/xml/user-include.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - ::/0 - - default - default - REPLACE_ME - 1 - - - - ::/0 - - default - default - REPLACE_ME - 1 - - diff --git a/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml b/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml deleted file mode 100644 index 903d7b6f733..00000000000 --- a/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml +++ /dev/null @@ -1 +0,0 @@ -network_max: 5000000 diff --git a/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml b/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml deleted file mode 100644 index 23b592507fa..00000000000 --- a/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml +++ /dev/null @@ -1,7 +0,0 @@ -test_user: - password: 'REPLACE_ME' - networks: - ip: '::/0' - profile: default - quota: default - access_management: 1 diff --git a/programs/diagnostics/testdata/configs/xml/config.xml b/programs/diagnostics/testdata/configs/xml/config.xml deleted file mode 100644 index eb7c70cf498..00000000000 --- a/programs/diagnostics/testdata/configs/xml/config.xml +++ /dev/null @@ -1,1195 +0,0 @@ - - - ../include/xml/server-include.xml - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - - 1000M - 10 - - - - - - - - - - - - - - - - - 8123 - - - 9000 - - - 9004 - - - 9005 - - - - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4096 - - - 3 - - - - - false - - - /path/to/ssl_cert_file - /path/to/ssl_key_file - - - false - - - /path/to/ssl_ca_cert_file - - - deflate - - - medium - - - -1 - -1 - - - false - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 100 - - - 0 - - - - 10000 - - - 0.9 - - - 4194304 - - - 0 - - - - - - 8589934592 - - - 5368709120 - - - - 1000 - - - 134217728 - - - 10000 - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - - - - /var/lib/clickhouse/user_files/ - - - - - - - - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - - - default - - - - - - - - - - - - default - - - - - - - - - true - - - false - - ' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - clickhouse-jdbc-bridge & - - * [CentOS/RHEL] - export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge - export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - clickhouse-jdbc-bridge & - - Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - ]]> - - - - - - - - REPLACE_ME - - - - - - - - localhost - 9000 - - - - - - - - - - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - - - - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - query_views_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - system - metric_log
- 7500 - 1000 - - - - - system - asynchronous_metric_log
- - 7000 - - - - - - - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - - system - opentelemetry_span_log
- 7500 - - - - - - system - crash_log
- - - 1000 - - - - - system - session_log
- - toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - - - - - *_dictionary.xml - - - *_function.xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - - hide encrypt/decrypt arguments - ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) - - \1(???) - - - - - - - - - - false - - false - - - https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 - - - - - - - - - - - s3 - https://storage.yandexcloud.net/my-bucket/root-path/ - REPLACE_ME - REPLACE_ME - -

Authorization: Bearer SOME-TOKEN
- your_base64_encoded_customer_key - - REPLACE_ME - REPLACE_ME - true - - http://proxy1 - http://proxy2 - - 10000 - 5000 - 10 - 4 - 1000 - /var/lib/clickhouse/disks/s3/ - false - - - - diff --git a/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml b/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml deleted file mode 100644 index 242a6a4b02e..00000000000 --- a/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - REPLACE_ME - - - \ No newline at end of file diff --git a/programs/diagnostics/testdata/configs/xml/users.xml b/programs/diagnostics/testdata/configs/xml/users.xml deleted file mode 100644 index cd5f17e922e..00000000000 --- a/programs/diagnostics/testdata/configs/xml/users.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - ../include/xml/user-include.xml - - - - - - 10000000000 - - random - 1 - - - - 1 - - - - - - - - - - REPLACE_ME - - ::/0 - - - default - - default - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/programs/diagnostics/testdata/configs/yaml/config.yaml b/programs/diagnostics/testdata/configs/yaml/config.yaml deleted file mode 100644 index e577a99e675..00000000000 --- a/programs/diagnostics/testdata/configs/yaml/config.yaml +++ /dev/null @@ -1,927 +0,0 @@ -# This is an example of a configuration file "config.xml" rewritten in YAML -# You can read this documentation for detailed information about YAML configuration: -# https://clickhouse.com/docs/en/operations/configuration-files/ - -# NOTE: User and query level settings are set up in "users.yaml" file. -# If you have accidentally specified user-level settings here, server won't start. -# You can either move the settings to the right place inside "users.xml" file -# or add skip_check_for_incorrect_settings: 1 here. -include_from: "../include/yaml/server-include.yaml" -logger: - # Possible levels [1]: - # - none (turns off logging) - # - fatal - # - critical - # - error - # - warning - # - notice - # - information - # - debug - # - trace - # [1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114 - level: trace - log: /var/log/clickhouse-server/clickhouse-server.log - errorlog: /var/log/clickhouse-server/clickhouse-server.err.log - # Rotation policy - # See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85 - size: 1000M - count: 10 - # console: 1 - # Default behavior is autodetection (log to console if not daemon mode and is tty) - - # Per level overrides (legacy): - # For example to suppress logging of the ConfigReloader you can use: - # NOTE: levels.logger is reserved, see below. - # levels: - # ConfigReloader: none - - # Per level overrides: - # For example to suppress logging of the RBAC for default user you can use: - # (But please note that the logger name maybe changed from version to version, even after minor upgrade) - # levels: - # - logger: - # name: 'ContextAccess (default)' - # level: none - # - logger: - # name: 'DatabaseOrdinary (test)' - # level: none - -# It is the name that will be shown in the clickhouse-client. -# By default, anything with "production" will be highlighted in red in query prompt. -# display_name: production - -# Port for HTTP API. See also 'https_port' for secure connections. -# This interface is also used by ODBC and JDBC drivers (DataGrip, Dbeaver, ...) -# and by most of web interfaces (embedded UI, Grafana, Redash, ...). -http_port: 8123 - -# Port for interaction by native protocol with: -# - clickhouse-client and other native ClickHouse tools (clickhouse-benchmark); -# - clickhouse-server with other clickhouse-servers for distributed query processing; -# - ClickHouse drivers and applications supporting native protocol -# (this protocol is also informally called as "the TCP protocol"); -# See also 'tcp_port_secure' for secure connections. -tcp_port: 9000 - -# Compatibility with MySQL protocol. -# ClickHouse will pretend to be MySQL for applications connecting to this port. -mysql_port: 9004 - -# Compatibility with PostgreSQL protocol. -# ClickHouse will pretend to be PostgreSQL for applications connecting to this port. -postgresql_port: 9005 - -# HTTP API with TLS (HTTPS). -# You have to configure certificate to enable this interface. -# See the openSSL section below. -# https_port: 8443 - -# Native interface with TLS. -# You have to configure certificate to enable this interface. -# See the openSSL section below. -# tcp_port_secure: 9440 - -# Native interface wrapped with PROXYv1 protocol -# PROXYv1 header sent for every connection. -# ClickHouse will extract information about proxy-forwarded client address from the header. -# tcp_with_proxy_port: 9011 - -# Port for communication between replicas. Used for data exchange. -# It provides low-level data access between servers. -# This port should not be accessible from untrusted networks. -# See also 'interserver_http_credentials'. -# Data transferred over connections to this port should not go through untrusted networks. -# See also 'interserver_https_port'. -interserver_http_port: 9009 - -# Port for communication between replicas with TLS. -# You have to configure certificate to enable this interface. -# See the openSSL section below. -# See also 'interserver_http_credentials'. -# interserver_https_port: 9010 - -# Hostname that is used by other replicas to request this server. -# If not specified, than it is determined analogous to 'hostname -f' command. -# This setting could be used to switch replication to another network interface -# (the server may be connected to multiple networks via multiple addresses) -# interserver_http_host: example.yandex.ru - -# You can specify credentials for authenthication between replicas. -# This is required when interserver_https_port is accessible from untrusted networks, -# and also recommended to avoid SSRF attacks from possibly compromised services in your network. -# interserver_http_credentials: -# user: interserver -# password: '' - -# Listen specified address. -# Use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere. -# Notes: -# If you open connections from wildcard address, make sure that at least one of the following measures applied: -# - server is protected by firewall and not accessible from untrusted networks; -# - all users are restricted to subset of network addresses (see users.xml); -# - all users have strong passwords, only secure (TLS) interfaces are accessible, or connections are only made via TLS interfaces. -# - users without password have readonly access. -# See also: https://www.shodan.io/search?query=clickhouse -# listen_host: '::' - -# Same for hosts without support for IPv6: -# listen_host: 0.0.0.0 - -# Default values - try listen localhost on IPv4 and IPv6. -# listen_host: '::1' -# listen_host: 127.0.0.1 - -# Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen. -# listen_try: 0 - -# Allow multiple servers to listen on the same address:port. This is not recommended. -# listen_reuse_port: 0 - -# listen_backlog: 64 -max_connections: 4096 - -# For 'Connection: keep-alive' in HTTP 1.1 -keep_alive_timeout: 3 - -# gRPC protocol (see src/Server/grpc_protos/clickhouse_grpc.proto for the API) -# grpc_port: 9100 -grpc: - enable_ssl: false - - # The following two files are used only if enable_ssl=1 - ssl_cert_file: /path/to/ssl_cert_file - ssl_key_file: /path/to/ssl_key_file - - # Whether server will request client for a certificate - ssl_require_client_auth: false - - # The following file is used only if ssl_require_client_auth=1 - ssl_ca_cert_file: /path/to/ssl_ca_cert_file - - # Default compression algorithm (applied if client doesn't specify another algorithm). - # Supported algorithms: none, deflate, gzip, stream_gzip - compression: deflate - - # Default compression level (applied if client doesn't specify another level). - # Supported levels: none, low, medium, high - compression_level: medium - - # Send/receive message size limits in bytes. -1 means unlimited - max_send_message_size: -1 - max_receive_message_size: -1 - - # Enable if you want very detailed logs - verbose_logs: false - -# Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -openSSL: - server: - # Used for https server AND secure tcp port - # openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt - certificateFile: /etc/clickhouse-server/server.crt - privateKeyFile: /etc/clickhouse-server/server.key - - # dhparams are optional. You can delete the dhParamsFile: element. - # To generate dhparams, use the following command: - # openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 - # Only file format with BEGIN DH PARAMETERS is supported. - dhParamsFile: /etc/clickhouse-server/dhparam.pem - verificationMode: none - loadDefaultCAFile: true - cacheSessions: true - disableProtocols: 'sslv2,sslv3' - preferServerCiphers: true - client: - # Used for connecting to https dictionary source and secured Zookeeper communication - loadDefaultCAFile: true - cacheSessions: true - disableProtocols: 'sslv2,sslv3' - preferServerCiphers: true - - # Use for self-signed: verificationMode: none - invalidCertificateHandler: - # Use for self-signed: name: AcceptCertificateHandler - name: RejectCertificateHandler - -# Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 -# http_server_default_response: |- -#
- -# Maximum number of concurrent queries. -max_concurrent_queries: 100 - -# Maximum memory usage (resident set size) for server process. -# Zero value or unset means default. Default is "max_server_memory_usage_to_ram_ratio" of available physical RAM. -# If the value is larger than "max_server_memory_usage_to_ram_ratio" of available physical RAM, it will be cut down. - -# The constraint is checked on query execution time. -# If a query tries to allocate memory and the current memory usage plus allocation is greater -# than specified threshold, exception will be thrown. - -# It is not practical to set this constraint to small values like just a few gigabytes, -# because memory allocator will keep this amount of memory in caches and the server will deny service of queries. -max_server_memory_usage: 0 - -# Maximum number of threads in the Global thread pool. -# This will default to a maximum of 10000 threads if not specified. -# This setting will be useful in scenarios where there are a large number -# of distributed queries that are running concurrently but are idling most -# of the time, in which case a higher number of threads might be required. -max_thread_pool_size: 10000 - -# On memory constrained environments you may have to set this to value larger than 1. -max_server_memory_usage_to_ram_ratio: 0.9 - -# Simple server-wide memory profiler. Collect a stack trace at every peak allocation step (in bytes). -# Data will be stored in system.trace_log table with query_id = empty string. -# Zero means disabled. -total_memory_profiler_step: 4194304 - -# Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. -# The probability is for every alloc/free regardless to the size of the allocation. -# Note that sampling happens only when the amount of untracked memory exceeds the untracked memory limit, -# which is 4 MiB by default but can be lowered if 'total_memory_profiler_step' is lowered. -# You may want to set 'total_memory_profiler_step' to 1 for extra fine grained sampling. -total_memory_tracker_sample_probability: 0 - -# Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve -# correct maximum value. -# max_open_files: 262144 - -# Size of cache of uncompressed blocks of data, used in tables of MergeTree family. -# In bytes. Cache is single for server. Memory is allocated only on demand. -# Cache is used when 'use_uncompressed_cache' user setting turned on (off by default). -# Uncompressed cache is advantageous only for very short queries and in rare cases. - -# Note: uncompressed cache can be pointless for lz4, because memory bandwidth -# is slower than multi-core decompression on some server configurations. -# Enabling it can sometimes paradoxically make queries slower. -uncompressed_cache_size: 8589934592 - -# Approximate size of mark cache, used in tables of MergeTree family. -# In bytes. Cache is single for server. Memory is allocated only on demand. -# You should not lower this value. -mark_cache_size: 5368709120 - -# If you enable the `min_bytes_to_use_mmap_io` setting, -# the data in MergeTree tables can be read with mmap to avoid copying from kernel to userspace. -# It makes sense only for large files and helps only if data reside in page cache. -# To avoid frequent open/mmap/munmap/close calls (which are very expensive due to consequent page faults) -# and to reuse mappings from several threads and queries, -# the cache of mapped files is maintained. Its size is the number of mapped regions (usually equal to the number of mapped files). -# The amount of data in mapped files can be monitored -# in system.metrics, system.metric_log by the MMappedFiles, MMappedFileBytes metrics -# and in system.asynchronous_metrics, system.asynchronous_metrics_log by the MMapCacheCells metric, -# and also in system.events, system.processes, system.query_log, system.query_thread_log, system.query_views_log by the -# CreatedReadBufferMMap, CreatedReadBufferMMapFailed, MMappedFileCacheHits, MMappedFileCacheMisses events. -# Note that the amount of data in mapped files does not consume memory directly and is not accounted -# in query or server memory usage - because this memory can be discarded similar to OS page cache. -# The cache is dropped (the files are closed) automatically on removal of old parts in MergeTree, -# also it can be dropped manually by the SYSTEM DROP MMAP CACHE query. -mmap_cache_size: 1000 - -# Cache size in bytes for compiled expressions. -compiled_expression_cache_size: 134217728 - -# Cache size in elements for compiled expressions. -compiled_expression_cache_elements_size: 10000 - -# Path to data directory, with trailing slash. -path: /var/lib/clickhouse/ - -# Path to temporary data for processing hard queries. -tmp_path: /var/lib/clickhouse/tmp/ - -# Policy from the for the temporary files. -# If not set is used, otherwise is ignored. - -# Notes: -# - move_factor is ignored -# - keep_free_space_bytes is ignored -# - max_data_part_size_bytes is ignored -# - you must have exactly one volume in that policy -# tmp_policy: tmp - -# Directory with user provided files that are accessible by 'file' table function. -user_files_path: /var/lib/clickhouse/user_files/ - -# LDAP server definitions. -ldap_servers: '' - -# List LDAP servers with their connection parameters here to later 1) use them as authenticators for dedicated local users, -# who have 'ldap' authentication mechanism specified instead of 'password', or to 2) use them as remote user directories. -# Parameters: -# host - LDAP server hostname or IP, this parameter is mandatory and cannot be empty. -# port - LDAP server port, default is 636 if enable_tls is set to true, 389 otherwise. -# bind_dn - template used to construct the DN to bind to. -# The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual -# user name during each authentication attempt. -# user_dn_detection - section with LDAP search parameters for detecting the actual user DN of the bound user. -# This is mainly used in search filters for further role mapping when the server is Active Directory. The -# resulting user DN will be used when replacing '{user_dn}' substrings wherever they are allowed. By default, -# user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected -# user DN value. -# base_dn - template used to construct the base DN for the LDAP search. -# The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings -# of the template with the actual user name and bind DN during the LDAP search. -# scope - scope of the LDAP search. -# Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). -# search_filter - template used to construct the search filter for the LDAP search. -# The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}' -# substrings of the template with the actual user name, bind DN, and base DN during the LDAP search. -# Note, that the special characters must be escaped properly in XML. -# verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed -# to be successfully authenticated for all consecutive requests without contacting the LDAP server. -# Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request. -# enable_tls - flag to trigger use of secure connection to the LDAP server. -# Specify 'no' for plain text (ldap://) protocol (not recommended). -# Specify 'yes' for LDAP over SSL/TLS (ldaps://) protocol (recommended, the default). -# Specify 'starttls' for legacy StartTLS protocol (plain text (ldap://) protocol, upgraded to TLS). -# tls_minimum_protocol_version - the minimum protocol version of SSL/TLS. -# Accepted values are: 'ssl2', 'ssl3', 'tls1.0', 'tls1.1', 'tls1.2' (the default). -# tls_require_cert - SSL/TLS peer certificate verification behavior. -# Accepted values are: 'never', 'allow', 'try', 'demand' (the default). -# tls_cert_file - path to certificate file. -# tls_key_file - path to certificate key file. -# tls_ca_cert_file - path to CA certificate file. -# tls_ca_cert_dir - path to the directory containing CA certificates. -# tls_cipher_suite - allowed cipher suite (in OpenSSL notation). -# Example: -# my_ldap_server: -# host: localhost -# port: 636 -# bind_dn: 'uid={user_name},ou=users,dc=example,dc=com' -# verification_cooldown: 300 -# enable_tls: yes -# tls_minimum_protocol_version: tls1.2 -# tls_require_cert: demand -# tls_cert_file: /path/to/tls_cert_file -# tls_key_file: /path/to/tls_key_file -# tls_ca_cert_file: /path/to/tls_ca_cert_file -# tls_ca_cert_dir: /path/to/tls_ca_cert_dir -# tls_cipher_suite: ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 - -# Example (typical Active Directory with configured user DN detection for further role mapping): -# my_ad_server: -# host: localhost -# port: 389 -# bind_dn: 'EXAMPLE\{user_name}' -# user_dn_detection: -# base_dn: CN=Users,DC=example,DC=com -# search_filter: '(&(objectClass=user)(sAMAccountName={user_name}))' -# enable_tls: no - -# To enable Kerberos authentication support for HTTP requests (GSS-SPNEGO), for those users who are explicitly configured -# to authenticate via Kerberos, define a single 'kerberos' section here. -# Parameters: -# principal - canonical service principal name, that will be acquired and used when accepting security contexts. -# This parameter is optional, if omitted, the default principal will be used. -# This parameter cannot be specified together with 'realm' parameter. -# realm - a realm, that will be used to restrict authentication to only those requests whose initiator's realm matches it. -# This parameter is optional, if omitted, no additional filtering by realm will be applied. -# This parameter cannot be specified together with 'principal' parameter. -# Example: -# kerberos: '' - -# Example: -# kerberos: -# principal: HTTP/clickhouse.example.com@EXAMPLE.COM - -# Example: -# kerberos: -# realm: EXAMPLE.COM - -# Sources to read users, roles, access rights, profiles of settings, quotas. -user_directories: - users_xml: - # Path to configuration file with predefined users. - path: users.yaml - local_directory: - # Path to folder where users created by SQL commands are stored. - path: /var/lib/clickhouse/access/ - -# # To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section -# # with the following parameters: -# # server - one of LDAP server names defined in 'ldap_servers' config section above. -# # This parameter is mandatory and cannot be empty. -# # roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. -# # If no roles are specified here or assigned during role mapping (below), user will not be able to perform any -# # actions after authentication. -# # role_mapping - section with LDAP search parameters and mapping rules. -# # When a user authenticates, while still bound to LDAP, an LDAP search is performed using search_filter and the -# # name of the logged in user. For each entry found during that search, the value of the specified attribute is -# # extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the -# # value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by -# # CREATE ROLE command. -# # There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be -# # applied. -# # base_dn - template used to construct the base DN for the LDAP search. -# # The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}' -# # substrings of the template with the actual user name, bind DN, and user DN during each LDAP search. -# # scope - scope of the LDAP search. -# # Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). -# # search_filter - template used to construct the search filter for the LDAP search. -# # The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and -# # '{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during -# # each LDAP search. -# # Note, that the special characters must be escaped properly in XML. -# # attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default. -# # prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by -# # the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated -# # as local role names. Empty, by default. -# # Example: -# # ldap: -# # server: my_ldap_server -# # roles: -# # my_local_role1: '' -# # my_local_role2: '' -# # role_mapping: -# # base_dn: 'ou=groups,dc=example,dc=com' -# # scope: subtree -# # search_filter: '(&(objectClass=groupOfNames)(member={bind_dn}))' -# # attribute: cn -# # prefix: clickhouse_ -# # Example (typical Active Directory with role mapping that relies on the detected user DN): -# # ldap: -# # server: my_ad_server -# # role_mapping: -# # base_dn: 'CN=Users,DC=example,DC=com' -# # attribute: CN -# # scope: subtree -# # search_filter: '(&(objectClass=group)(member={user_dn}))' -# # prefix: clickhouse_ - -# Default profile of settings. -default_profile: default - -# Comma-separated list of prefixes for user-defined settings. -# custom_settings_prefixes: '' -# System profile of settings. This settings are used by internal processes (Distributed DDL worker and so on). -# system_profile: default - -# Buffer profile of settings. -# This settings are used by Buffer storage to flush data to the underlying table. -# Default: used from system_profile directive. -# buffer_profile: default - -# Default database. -default_database: default - -# Server time zone could be set here. - -# Time zone is used when converting between String and DateTime types, -# when printing DateTime in text formats and parsing DateTime from text, -# it is used in date and time related functions, if specific time zone was not passed as an argument. - -# Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan. -# If not specified, system time zone at server startup is used. - -# Please note, that server could display time zone alias instead of specified name. -# Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC. -# timezone: Europe/Moscow - -# You can specify umask here (see "man umask"). Server will apply it on startup. -# Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read). -# umask: 022 - -# Perform mlockall after startup to lower first queries latency -# and to prevent clickhouse executable from being paged out under high IO load. -# Enabling this option is recommended but will lead to increased startup time for up to a few seconds. -mlock_executable: true - -# Reallocate memory for machine code ("text") using huge pages. Highly experimental. -remap_executable: false - -# Uncomment below in order to use JDBC table engine and function. -# To install and run JDBC bridge in background: -# * [Debian/Ubuntu] -# export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge -# export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') -# wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb -# apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb -# clickhouse-jdbc-bridge & -# * [CentOS/RHEL] -# export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge -# export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') -# wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm -# yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm -# clickhouse-jdbc-bridge & -# Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - -# jdbc_bridge: -# host: 127.0.0.1 -# port: 9019 - -# Configuration of clusters that could be used in Distributed tables. -# https://clickhouse.com/docs/en/operations/table_engines/distributed/ -remote_servers: - # Test only shard config for testing distributed storage - test_shard_localhost: - # Inter-server per-cluster secret for Distributed queries - # default: no secret (no authentication will be performed) - - # If set, then Distributed queries will be validated on shards, so at least: - # - such cluster should exist on the shard, - # - such cluster should have the same secret. - - # And also (and which is more important), the initial_user will - # be used as current user for the query. - - # Right now the protocol is pretty simple and it only takes into account: - # - cluster name - # - query - - # Also it will be nice if the following will be implemented: - # - source hostname (see interserver_http_host), but then it will depends from DNS, - # it can use IP address instead, but then the you need to get correct on the initiator node. - # - target hostname / ip address (same notes as for source hostname) - # - time-based security tokens - secret: 'REPLACE_ME' - shard: - # Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). - # internal_replication: false - # Optional. Shard weight when writing data. Default: 1. - # weight: 1 - replica: - host: localhost - port: 9000 - # Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). - # priority: 1 - -# The list of hosts allowed to use in URL-related storage engines and table functions. -# If this section is not present in configuration, all hosts are allowed. -# remote_url_allow_hosts: - -# Host should be specified exactly as in URL. The name is checked before DNS resolution. -# Example: "yandex.ru", "yandex.ru." and "www.yandex.ru" are different hosts. -# If port is explicitly specified in URL, the host:port is checked as a whole. -# If host specified here without port, any port with this host allowed. -# "yandex.ru" -> "yandex.ru:443", "yandex.ru:80" etc. is allowed, but "yandex.ru:80" -> only "yandex.ru:80" is allowed. -# If the host is specified as IP address, it is checked as specified in URL. Example: "[2a02:6b8:a::a]". -# If there are redirects and support for redirects is enabled, every redirect (the Location field) is checked. - -# Regular expression can be specified. RE2 engine is used for regexps. -# Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter -# (forgetting to do so is a common source of error). - -# If element has 'incl' attribute, then for it's value will be used corresponding substitution from another file. -# By default, path to file with substitutions is /etc/metrika.xml. It could be changed in config in 'include_from' element. -# Values for substitutions are specified in /clickhouse/name_of_substitution elements in that file. - -# ZooKeeper is used to store metadata about replicas, when using Replicated tables. -# Optional. If you don't use replicated tables, you could omit that. -# See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/ - -# zookeeper: -# - node: -# host: example1 -# port: 2181 -# - node: -# host: example2 -# port: 2181 -# - node: -# host: example3 -# port: 2181 - -# Substitutions for parameters of replicated tables. -# Optional. If you don't use replicated tables, you could omit that. -# See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables -# macros: -# shard: 01 -# replica: example01-01-1 - -# Reloading interval for embedded dictionaries, in seconds. Default: 3600. -builtin_dictionaries_reload_interval: 3600 - -# Maximum session timeout, in seconds. Default: 3600. -max_session_timeout: 3600 - -# Default session timeout, in seconds. Default: 60. -default_session_timeout: 60 - -# Sending data to Graphite for monitoring. Several sections can be defined. -# interval - send every X second -# root_path - prefix for keys -# hostname_in_path - append hostname to root_path (default = true) -# metrics - send data from table system.metrics -# events - send data from table system.events -# asynchronous_metrics - send data from table system.asynchronous_metrics - -# graphite: -# host: localhost -# port: 42000 -# timeout: 0.1 -# interval: 60 -# root_path: one_min -# hostname_in_path: true - -# metrics: true -# events: true -# events_cumulative: false -# asynchronous_metrics: true - -# graphite: -# host: localhost -# port: 42000 -# timeout: 0.1 -# interval: 1 -# root_path: one_sec - -# metrics: true -# events: true -# events_cumulative: false -# asynchronous_metrics: false - -# Serve endpoint for Prometheus monitoring. -# endpoint - mertics path (relative to root, statring with "/") -# port - port to setup server. If not defined or 0 than http_port used -# metrics - send data from table system.metrics -# events - send data from table system.events -# asynchronous_metrics - send data from table system.asynchronous_metrics - -# prometheus: -# endpoint: /metrics -# port: 9363 - -# metrics: true -# events: true -# asynchronous_metrics: true - -# Query log. Used only for queries with setting log_queries = 1. -query_log: - # What table to insert data. If table is not exist, it will be created. - # When query log structure is changed after system update, - # then old table will be renamed and new table will be created automatically. - database: system - table: query_log - - # PARTITION BY expr: https://clickhouse.com/docs/en/table_engines/mergetree-family/custom_partitioning_key/ - # Example: - # event_date - # toMonday(event_date) - # toYYYYMM(event_date) - # toStartOfHour(event_time) - partition_by: toYYYYMM(event_date) - - # Table TTL specification: https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl - # Example: - # event_date + INTERVAL 1 WEEK - # event_date + INTERVAL 7 DAY DELETE - # event_date + INTERVAL 2 WEEK TO DISK 'bbb' - - # ttl: 'event_date + INTERVAL 30 DAY DELETE' - - # Instead of partition_by, you can provide full engine expression (starting with ENGINE = ) with parameters, - # Example: engine: 'ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024' - - # Interval of flushing data. - flush_interval_milliseconds: 7500 - -# Trace log. Stores stack traces collected by query profilers. -# See query_profiler_real_time_period_ns and query_profiler_cpu_time_period_ns settings. -trace_log: - database: system - table: trace_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Query thread log. Has information about all threads participated in query execution. -# Used only for queries with setting log_query_threads = 1. -query_thread_log: - database: system - table: query_thread_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Query views log. Has information about all dependent views associated with a query. -# Used only for queries with setting log_query_views = 1. -query_views_log: - database: system - table: query_views_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Uncomment if use part log. -# Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads). -part_log: - database: system - table: part_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Uncomment to write text log into table. -# Text log contains all information from usual server log but stores it in structured and efficient way. -# The level of the messages that goes to the table can be limited (), if not specified all messages will go to the table. -# text_log: -# database: system -# table: text_log -# flush_interval_milliseconds: 7500 -# level: '' - -# Metric log contains rows with current values of ProfileEvents, CurrentMetrics collected with "collect_interval_milliseconds" interval. -metric_log: - database: system - table: metric_log - flush_interval_milliseconds: 7500 - collect_interval_milliseconds: 1000 - -# Asynchronous metric log contains values of metrics from -# system.asynchronous_metrics. -asynchronous_metric_log: - database: system - table: asynchronous_metric_log - - # Asynchronous metrics are updated once a minute, so there is - # no need to flush more often. - flush_interval_milliseconds: 60000 - -# OpenTelemetry log contains OpenTelemetry trace spans. -opentelemetry_span_log: - - # The default table creation code is insufficient, this spec - # is a workaround. There is no 'event_time' for this log, but two times, - # start and finish. It is sorted by finish time, to avoid inserting - # data too far away in the past (probably we can sometimes insert a span - # that is seconds earlier than the last span in the table, due to a race - # between several spans inserted in parallel). This gives the spans a - # global order that we can use to e.g. retry insertion into some external - # system. - engine: |- - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - database: system - table: opentelemetry_span_log - flush_interval_milliseconds: 7500 - -# Crash log. Stores stack traces for fatal errors. -# This table is normally empty. -crash_log: - database: system - table: crash_log - partition_by: '' - flush_interval_milliseconds: 1000 - -# Parameters for embedded dictionaries, used in Yandex.Metrica. -# See https://clickhouse.com/docs/en/dicts/internal_dicts/ - -# Path to file with region hierarchy. -# path_to_regions_hierarchy_file: /opt/geo/regions_hierarchy.txt - -# Path to directory with files containing names of regions -# path_to_regions_names_files: /opt/geo/ - - -# top_level_domains_path: /var/lib/clickhouse/top_level_domains/ -# Custom TLD lists. -# Format: name: /path/to/file - -# Changes will not be applied w/o server restart. -# Path to the list is under top_level_domains_path (see above). -top_level_domains_lists: '' - -# public_suffix_list: /path/to/public_suffix_list.dat - -# Configuration of external dictionaries. See: -# https://clickhouse.com/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts -dictionaries_config: '*_dictionary.xml' - -# Uncomment if you want data to be compressed 30-100% better. -# Don't do that if you just started using ClickHouse. - -# compression: -# # Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used. -# case: -# Conditions. All must be satisfied. Some conditions may be omitted. -# # min_part_size: 10000000000 # Min part size in bytes. -# # min_part_size_ratio: 0.01 # Min size of part relative to whole table size. -# # What compression method to use. -# method: zstd - -# Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster. -# Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -distributed_ddl: - # Path in ZooKeeper to queue with DDL queries - path: /clickhouse/task_queue/ddl - - # Settings from this profile will be used to execute DDL queries - # profile: default - - # Controls how much ON CLUSTER queries can be run simultaneously. - # pool_size: 1 - - # Cleanup settings (active tasks will not be removed) - - # Controls task TTL (default 1 week) - # task_max_lifetime: 604800 - - # Controls how often cleanup should be performed (in seconds) - # cleanup_delay_period: 60 - - # Controls how many tasks could be in the queue - # max_tasks_in_queue: 1000 - -# Settings to fine tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h -# merge_tree: -# max_suspicious_broken_parts: 5 - -# Protection from accidental DROP. -# If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query. -# If you want do delete one table and don't want to change clickhouse-server config, you could create special file /flags/force_drop_table and make DROP once. -# By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables. -# The same for max_partition_size_to_drop. -# Uncomment to disable protection. - -# max_table_size_to_drop: 0 -# max_partition_size_to_drop: 0 - -# Example of parameters for GraphiteMergeTree table engine -graphite_rollup_example: - pattern: - regexp: click_cost - function: any - retention: - - age: 0 - precision: 3600 - - age: 86400 - precision: 60 - default: - function: max - retention: - - age: 0 - precision: 60 - - age: 3600 - precision: 300 - - age: 86400 - precision: 3600 - -# Directory in containing schema files for various input formats. -# The directory will be created if it doesn't exist. -format_schema_path: /var/lib/clickhouse/format_schemas/ - -# Default query masking rules, matching lines would be replaced with something else in the logs -# (both text logs and system.query_log). -# name - name for the rule (optional) -# regexp - RE2 compatible regular expression (mandatory) -# replace - substitution string for sensitive data (optional, by default - six asterisks) -query_masking_rules: - rule: - name: hide encrypt/decrypt arguments - regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)' - # or more secure, but also more invasive: - # (aes_\w+)\s*\(.*\) - replace: \1(???) - -# Uncomment to use custom http handlers. -# rules are checked from top to bottom, first match runs the handler -# url - to match request URL, you can use 'regex:' prefix to use regex match(optional) -# methods - to match request method, you can use commas to separate multiple method matches(optional) -# headers - to match request headers, match each child element(child element name is header name), you can use 'regex:' prefix to use regex match(optional) -# handler is request handler -# type - supported types: static, dynamic_query_handler, predefined_query_handler -# query - use with predefined_query_handler type, executes query when the handler is called -# query_param_name - use with dynamic_query_handler type, extracts and executes the value corresponding to the value in HTTP request params -# status - use with static type, response status code -# content_type - use with static type, response content-type -# response_content - use with static type, Response content sent to client, when using the prefix 'file://' or 'config://', find the content from the file or configuration send to client. - -# http_handlers: -# - rule: -# url: / -# methods: POST,GET -# headers: -# pragma: no-cache -# handler: -# type: dynamic_query_handler -# query_param_name: query -# - rule: -# url: /predefined_query -# methods: POST,GET -# handler: -# type: predefined_query_handler -# query: 'SELECT * FROM system.settings' -# - rule: -# handler: -# type: static -# status: 200 -# content_type: 'text/plain; charset=UTF-8' -# response_content: config://http_server_default_response - -send_crash_reports: - # Changing to true allows sending crash reports to - # the ClickHouse core developers team via Sentry https://sentry.io - # Doing so at least in pre-production environments is highly appreciated - enabled: false - # Change to true if you don't feel comfortable attaching the server hostname to the crash report - anonymize: false - # Default endpoint should be changed to different Sentry DSN only if you have - # some in-house engineers or hired consultants who're going to debug ClickHouse issues for you - endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277' - # Uncomment to disable ClickHouse internal DNS caching. - # disable_internal_dns_cache: 1 - -storage_configuration: - disks: - s3: - secret_access_key: REPLACE_ME - access_key_id: 'REPLACE_ME' diff --git a/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml b/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml deleted file mode 100644 index c27bb7cb071..00000000000 --- a/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Users and ACL. -users: - # If user name was not specified, 'default' user is used. - default: - - password_sha256_hex: "REPLACE_ME" diff --git a/programs/diagnostics/testdata/configs/yaml/users.yaml b/programs/diagnostics/testdata/configs/yaml/users.yaml deleted file mode 100644 index 82f2d67f2a4..00000000000 --- a/programs/diagnostics/testdata/configs/yaml/users.yaml +++ /dev/null @@ -1,47 +0,0 @@ -include_from: "../include/yaml/user-include.yaml" -# Profiles of settings. -profiles: - # Default settings. - default: - # Maximum memory usage for processing single query, in bytes. - max_memory_usage: 10000000000 - load_balancing: random - - # Profile that allows only read queries. - readonly: - readonly: 1 - -# Users and ACL. -users: - # If user name was not specified, 'default' user is used. - default: - - password: 'REPLACE_ME' - - networks: - ip: '::/0' - - # Settings profile for user. - profile: default - - # Quota for user. - quota: default - - # User can create other users and grant rights to them. - # access_management: 1 - -# Quotas. -quotas: - # Name of quota. - default: - # Limits for time interval. You could specify many intervals with different limits. - interval: - # Length of interval. - duration: 3600 - - # No limits. Just calculate resource usage for time interval. - queries: 0 - errors: 0 - result_rows: 0 - read_rows: 0 - execution_time: 0 diff --git a/programs/diagnostics/testdata/configs/yandex_xml/config.xml b/programs/diagnostics/testdata/configs/yandex_xml/config.xml deleted file mode 100644 index 40d1fa34b1a..00000000000 --- a/programs/diagnostics/testdata/configs/yandex_xml/config.xml +++ /dev/null @@ -1,1167 +0,0 @@ - - - ../include/xml/server-include.xml - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - - 1000M - 10 - - - - - - - - - - - - - - - - - 8123 - - - 9000 - - - 9004 - - - 9005 - - - - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4096 - - - 3 - - - - - false - - - /path/to/ssl_cert_file - /path/to/ssl_key_file - - - false - - - /path/to/ssl_ca_cert_file - - - deflate - - - medium - - - -1 - -1 - - - false - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 100 - - - 0 - - - - 10000 - - - 0.9 - - - 4194304 - - - 0 - - - - - - 8589934592 - - - 5368709120 - - - - 1000 - - - 134217728 - - - 10000 - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - - - - /var/lib/clickhouse/user_files/ - - - - - - - - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - - - default - - - - - - - - - - - - default - - - - - - - - - true - - - false - - ' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - clickhouse-jdbc-bridge & - - * [CentOS/RHEL] - export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge - export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - clickhouse-jdbc-bridge & - - Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - ]]> - - - - - - - - - - - - - - - - localhost - 9000 - - - - - - - - - - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - - - - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - query_views_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - system - metric_log
- 7500 - 1000 - - - - - system - asynchronous_metric_log
- - 7000 - - - - - - - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - - system - opentelemetry_span_log
- 7500 - - - - - - system - crash_log
- - - 1000 - - - - - system - session_log
- - toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - - - - - *_dictionary.xml - - - *_function.xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - - hide encrypt/decrypt arguments - ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) - - \1(???) - - - - - - - - - - false - - false - - - https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 - - - - - - - - diff --git a/programs/diagnostics/testdata/docker/admin.xml b/programs/diagnostics/testdata/docker/admin.xml deleted file mode 100644 index 76aa670dcfe..00000000000 --- a/programs/diagnostics/testdata/docker/admin.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - 1 - - - - - 1 - - - \ No newline at end of file diff --git a/programs/diagnostics/testdata/docker/custom.xml b/programs/diagnostics/testdata/docker/custom.xml deleted file mode 100644 index bc1051178ca..00000000000 --- a/programs/diagnostics/testdata/docker/custom.xml +++ /dev/null @@ -1,8 +0,0 @@ - - :: - 0.0.0.0 - 1 - - 1 - - diff --git a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log deleted file mode 100644 index 1a1768fe87e..00000000000 --- a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log +++ /dev/null @@ -1,10 +0,0 @@ -2021.12.13 10:12:26.940169 [ 38398 ] {} Access(local directory): File /var/lib/clickhouse/access/users.list doesn't exist -2021.12.13 10:12:26.940204 [ 38398 ] {} Access(local directory): Recovering lists in directory /var/lib/clickhouse/access/ -2021.12.13 10:12:40.649453 [ 38445 ] {} Access(user directories): from: 127.0.0.1, user: default: Authentication failed: Code: 193. DB::Exception: Invalid credentials. (WRONG_PASSWORD), Stack trace (when copying this message, always include the lines below): - -0. DB::Exception::Exception(std::__1::basic_string, std::__1::allocator > const&, int, bool) @ 0x9b722d4 in /usr/bin/clickhouse -1. DB::IAccessStorage::throwInvalidCredentials() @ 0x119d9b27 in /usr/bin/clickhouse -2. DB::IAccessStorage::loginImpl(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&) const @ 0x119d98d7 in /usr/bin/clickhouse -3. DB::IAccessStorage::login(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&, bool) const @ 0x119d9084 in /usr/bin/clickhouse -4. DB::MultipleAccessStorage::loginImpl(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&) const @ 0x119ff93c in /usr/bin/clickhouse -5. DB::IAccessStorage::login(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&, bool) const @ 0x119d9084 in /usr/bin/clickhouse diff --git a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log deleted file mode 100644 index f6abe7764ba..00000000000 --- a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log +++ /dev/null @@ -1,10 +0,0 @@ -2022.02.02 14:49:32.458680 [ 200404 ] {} DiskLocal: Reserving 2.47 MiB on disk `default`, having unreserved 1.56 TiB. -2022.02.02 14:49:32.459086 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTask::PrepareStage: Merging 2 parts: from 202202_147058_147549_343 to 202202_147550_147550_0 into Wide -2022.02.02 14:49:32.459201 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTask::PrepareStage: Selected MergeAlgorithm: Horizontal -2022.02.02 14:49:32.459262 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTreeSequentialSource: Reading 159 marks from part 202202_147058_147549_343, total 1289014 rows starting from the beginning of the part -2022.02.02 14:49:32.459614 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTreeSequentialSource: Reading 2 marks from part 202202_147550_147550_0, total 2618 rows starting from the beginning of the part -2022.02.02 14:49:32.507755 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTask::MergeProjectionsStage: Merge sorted 1291632 rows, containing 5 columns (5 merged, 0 gathered) in 0.048711404 sec., 26516008.448452853 rows/sec., 639.52 MiB/sec. -2022.02.02 14:49:32.508332 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} system.asynchronous_metric_log (de87df8b-2250-439c-9e87-df8b2250339c): Renaming temporary part tmp_merge_202202_147058_147550_344 to 202202_147058_147550_344. -2022.02.02 14:49:32.508406 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} system.asynchronous_metric_log (de87df8b-2250-439c-9e87-df8b2250339c) (MergerMutator): Merged 2 parts: from 202202_147058_147549_343 to 202202_147550_147550_0 -2022.02.02 14:49:32.508440 [ 200359 ] {} MemoryTracker: Peak memory usage Mutate/Merge: 16.31 MiB. -2022.02.02 14:49:33.000148 [ 200388 ] {} AsynchronousMetrics: MemoryTracking: was 774.16 MiB, peak 2.51 GiB, will set to 772.30 MiB (RSS), difference: -1.86 MiB diff --git a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz deleted file mode 100644 index 136bf5913aa..00000000000 --- a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz +++ /dev/null @@ -1 +0,0 @@ -dummy hz file for tests From ada4384deb5fe0c4dd95459da9ac8a225cf92a51 Mon Sep 17 00:00:00 2001 From: liuneng <1398775315@qq.com> Date: Thu, 21 Mar 2024 10:38:51 +0800 Subject: [PATCH 0322/1165] fix ut failed --- src/Analyzer/Passes/ConvertInToEqualPass.cpp | 16 +++++++++++----- ...est_transform_query_for_external_database.cpp | 6 ++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp index fbac09de7aa..ddd8a8f266f 100644 --- a/src/Analyzer/Passes/ConvertInToEqualPass.cpp +++ b/src/Analyzer/Passes/ConvertInToEqualPass.cpp @@ -32,7 +32,8 @@ public: if (!column_node || !constant_node) return ; // IN multiple values is not supported - if (constant_node->getValue().getType() == Field::Types::Which::Tuple) + if (constant_node->getValue().getType() == Field::Types::Which::Tuple + || constant_node->getValue().getType() == Field::Types::Which::Array) return ; // x IN null not equivalent to x = null if (constant_node->hasSourceExpression() || constant_node->getValue().isNull()) @@ -51,12 +52,17 @@ public: { resolver = createInternalFunctionNotEqualOverloadResolver(decimal_check_overflow); } - equal->resolveAsFunction(resolver); + try + { + equal->resolveAsFunction(resolver); + } + catch (...) + { + // When function resolver fails, we should not replace the function node + return; + } node = equal; } -private: - FunctionOverloadResolverPtr equal_resolver; - FunctionOverloadResolverPtr not_equal_resolver; }; void ConvertInToEqualPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 7e2d393c3d1..6490498d717 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -306,7 +306,8 @@ TEST(TransformQueryForExternalDatabase, Aliases) check(state, 1, {"field"}, "SELECT field AS value, field AS display FROM table WHERE field NOT IN ('') AND display LIKE '%test%'", - R"(SELECT "field" FROM "test"."table" WHERE ("field" NOT IN ('')) AND ("field" LIKE '%test%'))"); + R"(SELECT "field" FROM "test"."table" WHERE ("field" NOT IN ('')) AND ("field" LIKE '%test%'))", + R"(SELECT "field" FROM "test"."table" WHERE ("field" != '') AND ("field" LIKE '%test%'))"); } TEST(TransformQueryForExternalDatabase, ForeignColumnInWhere) @@ -408,5 +409,6 @@ TEST(TransformQueryForExternalDatabase, Analyzer) check(state, 1, {"column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo"}, "SELECT * FROM table WHERE (column) IN (1)", - R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" IN (1))"); + R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" IN (1))", + R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" = 1)"); } From 8d9a58c93614cfc01bbd93765e3e01dc26da4e76 Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Thu, 21 Mar 2024 02:17:58 +0000 Subject: [PATCH 0323/1165] LowCardinalityFixedStringColumn: allow generic types Fix tests and add new ones. --- src/Functions/FunctionsJSON.h | 9 ++------- .../02474_extract_fixedstring_from_json.reference | 7 +++++++ .../0_stateless/02474_extract_fixedstring_from_json.sql | 7 +++++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 53515985e39..8a193785f87 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -865,11 +865,9 @@ struct JSONExtractTree explicit LowCardinalityFixedStringNode(const size_t fixed_length_) : fixed_length(fixed_length_) { } bool insertResultToColumn(IColumn & dest, const Element & element) override { - // If element is an object we delegate the insertion to JSONExtractRawImpl - if (element.isObject()) + // For types other than string, delegate the insertion to JSONExtractRawImpl. + if (!element.isString()) return JSONExtractRawImpl::insertResultToLowCardinalityFixedStringColumn(dest, element, fixed_length); - else if (!element.isString()) - return false; auto str = element.getString(); if (str.size() > fixed_length) @@ -1484,9 +1482,6 @@ public: // We use insertResultToLowCardinalityFixedStringColumn in case we are inserting raw data in a Low Cardinality FixedString column static bool insertResultToLowCardinalityFixedStringColumn(IColumn & dest, const Element & element, size_t fixed_length) { - if (element.getObject().size() > fixed_length) - return false; - ColumnFixedString::Chars chars; WriteBufferFromVector buf(chars, AppendModeTag()); traverse(element, buf); diff --git a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference index 783d12fcf1a..21ddf5d3512 100644 --- a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference +++ b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference @@ -8,3 +8,10 @@ \0\0\0\0\0 131231 131231 +1234 +1234 +{"b":131231} +\0\0\0\0 +1234567890 +18446744073709551615 +-9223372036854775807 diff --git a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql index cfc47e00cba..bbb9f55062b 100644 --- a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql +++ b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql @@ -6,3 +6,10 @@ SELECT JSONExtract('{"a": 123456}', 'a', 'FixedString(5)'); SELECT JSONExtract('{"a": 123456}', 'a', 'FixedString(6)'); SELECT JSONExtract(materialize('{"a": 131231}'), 'a', 'LowCardinality(FixedString(5))') FROM numbers(2); SELECT JSONExtract(materialize('{"a": 131231}'), 'a', 'LowCardinality(FixedString(6))') FROM numbers(2); +SELECT JSONExtract(materialize('{"a": 131231, "b": 1234}'), 'b', 'LowCardinality(FixedString(4))'); +SELECT JSONExtract(materialize('{"a": 131231, "b": "1234"}'), 'b', 'LowCardinality(FixedString(4))'); +SELECT JSONExtract(materialize('{"a": {"b": 131231} }'), 'a', 'LowCardinality(FixedString(12))'); +SELECT JSONExtract(materialize('{"a": 131231, "b": 1234567890}'), 'b', 'LowCardinality(FixedString(4))'); +SELECT JSONExtract(materialize('{"a": 131231, "b": 1234567890}'), 'b', 'LowCardinality(FixedString(10))'); +SELECT JSONExtract(materialize('{"a": 18446744073709551615}'), 'a', 'LowCardinality(FixedString(20))'); +SELECT JSONExtract(materialize('{"a": -9223372036854775807}'), 'a', 'LowCardinality(FixedString(20))'); From adc964568c29e69131c37e600c867a98957c65da Mon Sep 17 00:00:00 2001 From: unashi Date: Thu, 21 Mar 2024 15:44:27 +0800 Subject: [PATCH 0324/1165] [update] Merge the on same disk and on another disk scenarios into cloneAndLoadDataPart; remove the try catch; instead of obtaining the copied destination hard disk through traversal, the destination hard disk is obtained through reservation. --- src/Storages/MergeTree/MergeTreeData.cpp | 166 ++------------------ src/Storages/MergeTree/MergeTreeData.h | 11 +- src/Storages/MergeTree/MutateTask.cpp | 2 +- src/Storages/StorageMergeTree.cpp | 43 ++--- src/Storages/StorageReplicatedMergeTree.cpp | 48 ++---- 5 files changed, 42 insertions(+), 228 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index f8448be7268..aed2db16504 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7063,7 +7063,7 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData( return checkStructureAndGetMergeTreeData(*source_table, src_snapshot, my_snapshot); } -std::pair MergeTreeData::cloneAndLoadDataPartOnSameDisk( +std::pair MergeTreeData::cloneAndLoadDataPart( const MergeTreeData::DataPartPtr & src_part, const String & tmp_part_prefix, const MergeTreePartInfo & dst_part_info, @@ -7073,22 +7073,16 @@ std::pair MergeTreeData::cloneAn const WriteSettings & write_settings) { chassert(!isStaticStorage()); - - /// Check that the storage policy contains the disk where the src_part is located. - bool does_storage_policy_allow_same_disk = false; - for (const DiskPtr & disk : getStoragePolicy()->getDisks()) + bool on_same_disk = false; + for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) { if (disk->getName() == src_part->getDataPartStorage().getDiskName()) { - does_storage_policy_allow_same_disk = true; + on_same_disk = true; break; } } - if (!does_storage_policy_allow_same_disk) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Could not clone and load part {} because disk does not belong to storage policy", - quoteString(src_part->getDataPartStorage().getFullPath())); + String dst_part_name = src_part->getNewName(dst_part_info); String tmp_dst_part_name = tmp_part_prefix + dst_part_name; @@ -7103,11 +7097,12 @@ std::pair MergeTreeData::cloneAn MergeTreeData::MutableDataPartPtr src_flushed_tmp_part; String with_copy; - if (params.copy_instead_of_hardlink) + if (params.copy_instead_of_hardlink || !on_same_disk) with_copy = " (copying data)"; + std::shared_ptr dst_part_storage{}; - try + if (on_same_disk && !params.copy_instead_of_hardlink) { dst_part_storage = src_part_storage->freeze( relative_data_path, @@ -7117,34 +7112,13 @@ std::pair MergeTreeData::cloneAn /* save_metadata_callback= */ {}, params); } - catch (...) + else { - /// Hardlink fail. Try copy. - LOG_WARNING( - &Poco::Logger::get("MergeTreeData"), - "Hard link fail, try tp copy directly. to:{}, path:{}", - this->getRelativeDataPath(), - tmp_dst_part_name); - bool copy_successful = false; - for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) - { - try - { - auto reservation_space = src_part_storage->reserve(src_part->getBytesOnDisk()); - if (!reservation_space) - throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Not enough space on disk."); - dst_part_storage = src_part_storage->clonePart( - this->getRelativeDataPath(), tmp_dst_part_name, disk, read_settings, write_settings, {}, {}); - copy_successful = true; - break; - } - catch (Exception & e) - { - LOG_TRACE(&Poco::Logger::get("MergeTreeData"), "Clone part on disk {} fail: {}", disk->getName(), e.what()); - } - } - if (!copy_successful) - LOG_ERROR(&Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail."); + auto reservation_on_dst = getStoragePolicy()->reserve(src_part->getBytesOnDisk()); + if (!reservation_on_dst) + throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Not enough space on disk."); + dst_part_storage = src_part_storage->clonePart( + this->getRelativeDataPath(), tmp_dst_part_name, reservation_on_dst->getDisk(), read_settings, write_settings, {}, {}); } @@ -7168,117 +7142,7 @@ std::pair MergeTreeData::cloneAn .withPartFormatFromDisk() .build(); - if (!params.copy_instead_of_hardlink && params.hardlinked_files) - { - params.hardlinked_files->source_part_name = src_part->name; - params.hardlinked_files->source_table_shared_id = src_part->storage.getTableSharedID(); - - for (auto it = src_part->getDataPartStorage().iterate(); it->isValid(); it->next()) - { - if (!params.files_to_copy_instead_of_hardlinks.contains(it->name()) - && it->name() != IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME_DEPRECATED - && it->name() != IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME) - { - params.hardlinked_files->hardlinks_from_source_part.insert(it->name()); - } - } - - auto projections = src_part->getProjectionParts(); - for (const auto & [name, projection_part] : projections) - { - const auto & projection_storage = projection_part->getDataPartStorage(); - for (auto it = projection_storage.iterate(); it->isValid(); it->next()) - { - auto file_name_with_projection_prefix = fs::path(projection_storage.getPartDirectory()) / it->name(); - if (!params.files_to_copy_instead_of_hardlinks.contains(file_name_with_projection_prefix) - && it->name() != IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME_DEPRECATED - && it->name() != IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME) - { - params.hardlinked_files->hardlinks_from_source_part.insert(file_name_with_projection_prefix); - } - } - } - } - - /// We should write version metadata on part creation to distinguish it from parts that were created without transaction. - TransactionID tid = params.txn ? params.txn->tid : Tx::PrehistoricTID; - dst_data_part->version.setCreationTID(tid, nullptr); - dst_data_part->storeVersionMetadata(); - - dst_data_part->is_temp = true; - - dst_data_part->loadColumnsChecksumsIndexes(require_part_metadata, true); - dst_data_part->modification_time = dst_part_storage->getLastModified().epochTime(); - return std::make_pair(dst_data_part, std::move(temporary_directory_lock)); -} - -std::pair MergeTreeData::cloneAndLoadDataPartOnOtherDisk( - const MergeTreeData::DataPartPtr & src_part, - const String & tmp_part_prefix, - const MergeTreePartInfo & dst_part_info, - const StorageMetadataPtr & metadata_snapshot, - const IDataPartStorage::ClonePartParams & params, - const ReadSettings & read_settings, - const WriteSettings & write_settings) -{ - chassert(!isStaticStorage()); - - String dst_part_name = src_part->getNewName(dst_part_info); - String tmp_dst_part_name = tmp_part_prefix + dst_part_name; - auto temporary_directory_lock = getTemporaryPartDirectoryHolder(tmp_dst_part_name); - - auto reservation = src_part->getDataPartStorage().reserve(src_part->getBytesOnDisk()); - auto src_part_storage = src_part->getDataPartStoragePtr(); - - scope_guard src_flushed_tmp_dir_lock; - MergeTreeData::MutableDataPartPtr src_flushed_tmp_part; - - String with_copy; - if (params.copy_instead_of_hardlink) - with_copy = " (copying data)"; - - std::shared_ptr dst_part_storage{}; - bool copy_successful = false; - for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) - { - try - { - auto reservation_space = src_part_storage->reserve(src_part->getBytesOnDisk()); - if (!reservation_space) - throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Not enough space on disk."); - dst_part_storage - = src_part_storage->clonePart(this->getRelativeDataPath(), tmp_dst_part_name, disk, read_settings, write_settings, {}, {}); - copy_successful = true; - break; - } - catch (...) - { - LOG_TRACE(&Poco::Logger::get("MergeTreeData"), "Clone part on disk {} fail", disk->getName()); - } - } - if (!copy_successful) - LOG_FATAL(&Poco::Logger::get("MergeTreeData"), "Hard link fail, clone fail."); - if (params.metadata_version_to_write.has_value()) - { - chassert(!params.keep_metadata_version); - auto out_metadata = dst_part_storage->writeFile(IMergeTreeDataPart::METADATA_VERSION_FILE_NAME, 4096, getContext()->getWriteSettings()); - writeText(metadata_snapshot->getMetadataVersion(), *out_metadata); - out_metadata->finalize(); - if (getSettings()->fsync_after_insert) - out_metadata->sync(); - } - - LOG_DEBUG(log, "Clone{} part {} to {}{}", - src_flushed_tmp_part ? " flushed" : "", - src_part_storage->getFullPath(), - std::string(fs::path(dst_part_storage->getFullRootPath()) / tmp_dst_part_name), - with_copy); - - auto dst_data_part = MergeTreeDataPartBuilder(*this, dst_part_name, dst_part_storage) - .withPartFormatFromDisk() - .build(); - - if (!params.copy_instead_of_hardlink && params.hardlinked_files) + if (on_same_disk && !params.copy_instead_of_hardlink && params.hardlinked_files) { params.hardlinked_files->source_part_name = src_part->name; params.hardlinked_files->source_table_shared_id = src_part->storage.getTableSharedID(); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 95bec1eeb2b..9b9e5f97f36 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -834,16 +834,7 @@ public: MergeTreeData & checkStructureAndGetMergeTreeData(const StoragePtr & source_table, const StorageMetadataPtr & src_snapshot, const StorageMetadataPtr & my_snapshot) const; MergeTreeData & checkStructureAndGetMergeTreeData(IStorage & source_table, const StorageMetadataPtr & src_snapshot, const StorageMetadataPtr & my_snapshot) const; - std::pair cloneAndLoadDataPartOnSameDisk( - const MergeTreeData::DataPartPtr & src_part, - const String & tmp_part_prefix, - const MergeTreePartInfo & dst_part_info, - const StorageMetadataPtr & metadata_snapshot, - const IDataPartStorage::ClonePartParams & params, - const ReadSettings & read_settings, - const WriteSettings & write_settings); - - std::pair cloneAndLoadDataPartOnOtherDisk( + std::pair cloneAndLoadDataPart( const MergeTreeData::DataPartPtr & src_part, const String & tmp_part_prefix, const MergeTreePartInfo & dst_part_info, diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 150cc27c369..3ac103824bd 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -2097,7 +2097,7 @@ bool MutateTask::prepare() scope_guard lock; { - std::tie(part, lock) = ctx->data->cloneAndLoadDataPartOnSameDisk( + std::tie(part, lock) = ctx->data->cloneAndLoadDataPart( ctx->source_part, prefix, ctx->future_part->part_info, ctx->metadata_snapshot, clone_params, ctx->context->getReadSettings(), ctx->context->getWriteSettings()); part->getDataPartStorage().beginTransaction(); ctx->temporary_directory_lock = std::move(lock); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 89857156701..6adfc860cbc 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2099,37 +2099,16 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con MergeTreePartInfo dst_part_info(partition_id, temp_index, temp_index, src_part->info.level); IDataPartStorage::ClonePartParams clone_params{.txn = local_context->getCurrentTransaction()}; - bool on_same_disk = false; - for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) - if (disk->getName() == src_part->getDataPartStorage().getDiskName()) - on_same_disk = true; - if (on_same_disk && !clone_params.copy_instead_of_hardlink) - { - - auto [dst_part, part_lock] = cloneAndLoadDataPartOnSameDisk( - src_part, - TMP_PREFIX, - dst_part_info, - my_metadata_snapshot, - clone_params, - local_context->getReadSettings(), - local_context->getWriteSettings()); - dst_parts.emplace_back(std::move(dst_part)); - dst_parts_locks.emplace_back(std::move(part_lock)); - } - else - { - auto [dst_part, part_lock] = cloneAndLoadDataPartOnOtherDisk( - src_part, - TMP_PREFIX, - dst_part_info, - my_metadata_snapshot, - clone_params, - local_context->getReadSettings(), - local_context->getWriteSettings()); - dst_parts.emplace_back(std::move(dst_part)); - dst_parts_locks.emplace_back(std::move(part_lock)); - } + auto [dst_part, part_lock] = cloneAndLoadDataPart( + src_part, + TMP_PREFIX, + dst_part_info, + my_metadata_snapshot, + clone_params, + local_context->getReadSettings(), + local_context->getWriteSettings()); + dst_parts.emplace_back(std::move(dst_part)); + dst_parts_locks.emplace_back(std::move(part_lock)); } /// ATTACH empty part set @@ -2231,7 +2210,7 @@ void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const .copy_instead_of_hardlink = getSettings()->always_use_copy_instead_of_hardlinks, }; - auto [dst_part, part_lock] = dest_table_storage->cloneAndLoadDataPartOnSameDisk( + auto [dst_part, part_lock] = dest_table_storage->cloneAndLoadDataPart( src_part, TMP_PREFIX, dst_part_info, diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index fdbd2d47fe4..cf9cc6f27e1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2751,7 +2751,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) auto obtain_part = [&] (PartDescriptionPtr & part_desc) { - /// Fetches with zero-copy-replication are cheap, but cloneAndLoadDataPartOnSameDisk will do full copy. + /// Fetches with zero-copy-replication are cheap, but cloneAndLoadDataPart(OnSameDisk) will do full copy. /// It's okay to check the setting for current table and disk for the source table, because src and dst part are on the same disk. bool prefer_fetch_from_other_replica = !part_desc->replica.empty() && storage_settings_ptr->allow_remote_fs_zero_copy_replication && part_desc->src_table_part && part_desc->src_table_part->isStoredOnRemoteDiskWithZeroCopySupport(); @@ -2770,7 +2770,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) .copy_instead_of_hardlink = storage_settings_ptr->always_use_copy_instead_of_hardlinks || ((our_zero_copy_enabled || source_zero_copy_enabled) && part_desc->src_table_part->isStoredOnRemoteDiskWithZeroCopySupport()), .metadata_version_to_write = metadata_snapshot->getMetadataVersion() }; - auto [res_part, temporary_part_lock] = cloneAndLoadDataPartOnSameDisk( + auto [res_part, temporary_part_lock] = cloneAndLoadDataPart( part_desc->src_table_part, TMP_PREFIX + "clone_", part_desc->new_part_info, @@ -4847,7 +4847,7 @@ bool StorageReplicatedMergeTree::fetchPart( .keep_metadata_version = true, }; - auto [cloned_part, lock] = cloneAndLoadDataPartOnSameDisk( + auto [cloned_part, lock] = cloneAndLoadDataPart( part_to_clone, "tmp_clone_", part_info, @@ -8023,36 +8023,16 @@ void StorageReplicatedMergeTree::replacePartitionFrom( .copy_instead_of_hardlink = storage_settings_ptr->always_use_copy_instead_of_hardlinks || (zero_copy_enabled && src_part->isStoredOnRemoteDiskWithZeroCopySupport()), .metadata_version_to_write = metadata_snapshot->getMetadataVersion() }; - bool on_same_disk = false; - for (const DiskPtr & disk : this->getStoragePolicy()->getDisks()) - if (disk->getName() == src_part->getDataPartStorage().getDiskName()) - on_same_disk = true; - if (on_same_disk && !clone_params.copy_instead_of_hardlink) - { - auto [dst_part, part_lock] = cloneAndLoadDataPartOnSameDisk( - src_part, - TMP_PREFIX, - dst_part_info, - metadata_snapshot, - clone_params, - query_context->getReadSettings(), - query_context->getWriteSettings()); - dst_parts.emplace_back(std::move(dst_part)); - dst_parts_locks.emplace_back(std::move(part_lock)); - } - else - { - auto [dst_part, part_lock] = cloneAndLoadDataPartOnOtherDisk( - src_part, - TMP_PREFIX, - dst_part_info, - metadata_snapshot, - clone_params, - query_context->getReadSettings(), - query_context->getWriteSettings()); - dst_parts.emplace_back(std::move(dst_part)); - dst_parts_locks.emplace_back(std::move(part_lock)); - } + auto [dst_part, part_lock] = cloneAndLoadDataPart( + src_part, + TMP_PREFIX, + dst_part_info, + metadata_snapshot, + clone_params, + query_context->getReadSettings(), + query_context->getWriteSettings()); + dst_parts.emplace_back(std::move(dst_part)); + dst_parts_locks.emplace_back(std::move(part_lock)); src_parts.emplace_back(src_part); ephemeral_locks.emplace_back(std::move(*lock)); block_id_paths.emplace_back(block_id_path); @@ -8291,7 +8271,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta .copy_instead_of_hardlink = storage_settings_ptr->always_use_copy_instead_of_hardlinks || (zero_copy_enabled && src_part->isStoredOnRemoteDiskWithZeroCopySupport()), .metadata_version_to_write = dest_metadata_snapshot->getMetadataVersion() }; - auto [dst_part, dst_part_lock] = dest_table_storage->cloneAndLoadDataPartOnSameDisk( + auto [dst_part, dst_part_lock] = dest_table_storage->cloneAndLoadDataPart( src_part, TMP_PREFIX, dst_part_info, From 758b07db3c9eebea354e1f20f6d14b4102d1d3d8 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 08:45:35 +0100 Subject: [PATCH 0325/1165] Update 02967_parallel_replicas_join_algo_and_analyzer #ci_set_analyzer --- ..._replicas_join_algo_and_analyzer.reference | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference index fa343571ba0..165504174d0 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference @@ -25,8 +25,8 @@ simple (global) join with analyzer and parallel replicas SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState) SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState) DefaultCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, allow_experimental_parallel_reading_from_replicas = 2, send_logs_level = 'trace', max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, allow_experimental_parallel_reading_from_replicas = 2, send_logs_level = 'trace', max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done simple (local) join with analyzer and parallel replicas @@ -40,8 +40,8 @@ simple (local) join with analyzer and parallel replicas 4200042 4200042 4200042 -1400014 4200048 4200048 4200048 -1400016 4200054 4200054 4200054 -1400018 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done simple (local) join with analyzer and parallel replicas and full sorting merge join @@ -55,8 +55,8 @@ simple (local) join with analyzer and parallel replicas and full sorting merge j 4200042 4200042 4200042 -1400014 4200048 4200048 4200048 -1400016 4200054 4200054 4200054 -1400018 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, join_algorithm = 'full_sorting_merge', send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, join_algorithm = 'full_sorting_merge', send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer @@ -82,8 +82,8 @@ nested join with analyzer and parallel replicas, both local 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, join_algorithm = 'full_sorting_merge', send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, join_algorithm = 'full_sorting_merge', send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both global @@ -97,11 +97,11 @@ nested join with analyzer and parallel replicas, both global 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done nested join with analyzer and parallel replicas, global + local @@ -115,11 +115,11 @@ nested join with analyzer and parallel replicas, global + local 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting merge join @@ -133,11 +133,11 @@ nested join with analyzer and parallel replicas, both local, both full sorting m 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting and hash join @@ -151,11 +151,11 @@ nested join with analyzer and parallel replicas, both local, both full sorting a 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'hash' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'hash' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting and hash join @@ -169,9 +169,9 @@ nested join with analyzer and parallel replicas, both local, both full sorting a 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'hash' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'hash' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done From 7a3ab461bfe88b6d073ac8284ac5f12841b120cf Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 21 Mar 2024 09:24:35 +0100 Subject: [PATCH 0326/1165] Fix style check --- src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 5941dca0b48..3555c71abee 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -29,7 +29,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } - ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage( std::shared_ptr blob_container_client_, const String & path_, @@ -58,7 +57,6 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage( } } - void ReadBufferFromAzureBlobStorage::setReadUntilEnd() { if (read_until_position) @@ -141,7 +139,6 @@ bool ReadBufferFromAzureBlobStorage::nextImpl() return true; } - off_t ReadBufferFromAzureBlobStorage::seek(off_t offset_, int whence) { if (offset_ == getPosition() && whence == SEEK_SET) @@ -195,13 +192,11 @@ off_t ReadBufferFromAzureBlobStorage::seek(off_t offset_, int whence) return offset; } - off_t ReadBufferFromAzureBlobStorage::getPosition() { return offset - available(); } - void ReadBufferFromAzureBlobStorage::initialize() { if (initialized) @@ -279,8 +274,6 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran sleep_time_with_backoff_milliseconds *= 2; }; - - for (size_t i = 0; i < max_single_download_retries && n > 0; ++i) { size_t bytes_copied = 0; From 42d59a8fd11a8a66dd8392ecb83eb1971b8cfb1f Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 20 Mar 2024 17:05:34 +0000 Subject: [PATCH 0327/1165] Update after refactoring ALIAS columns computation --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 96 +++++++++++++++++++ .../0_stateless/00722_inner_join.reference | 6 +- .../02835_join_step_explain.reference | 36 +++---- ...alyzer_using_functional_args.reference.j2} | 11 ++- ...955_analyzer_using_functional_args.sql.j2} | 15 ++- 5 files changed, 135 insertions(+), 29 deletions(-) rename tests/queries/0_stateless/{02955_analyzer_using_functional_args.reference => 02955_analyzer_using_functional_args.reference.j2} (94%) rename tests/queries/0_stateless/{02955_analyzer_using_functional_args.sql => 02955_analyzer_using_functional_args.sql.j2} (86%) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 0414ea93f94..6a5efa6532f 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1392,6 +1392,8 @@ private: const NamesAndTypes & matched_columns, const IdentifierResolveScope & scope); + void updateMatchedColumnsFromJoinUsing(QueryTreeNodesWithNames & result_matched_column_nodes_with_names, const QueryTreeNodePtr & source_table_expression, IdentifierResolveScope & scope); + QueryTreeNodesWithNames resolveQualifiedMatcher(QueryTreeNodePtr & matcher_node, IdentifierResolveScope & scope); QueryTreeNodesWithNames resolveUnqualifiedMatcher(QueryTreeNodePtr & matcher_node, IdentifierResolveScope & scope); @@ -3543,9 +3545,14 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo if (scope.join_use_nulls) { + auto it = node_to_projection_name.find(resolved_identifier); auto nullable_resolved_identifier = convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side, scope); if (nullable_resolved_identifier) + { resolved_identifier = nullable_resolved_identifier; + if (it != node_to_projection_name.end()) + node_to_projection_name.emplace(resolved_identifier, it->second); + } } return resolved_identifier; @@ -4204,6 +4211,93 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::getMatchedColumnNodesWithN return matched_column_nodes_with_names; } +bool hasTableExpressionInJoinTree(const QueryTreeNodePtr & join_tree_node, const QueryTreeNodePtr & table_expression) +{ + QueryTreeNodes nodes_to_process; + nodes_to_process.push_back(join_tree_node); + + while (!nodes_to_process.empty()) + { + auto node_to_process = std::move(nodes_to_process.back()); + nodes_to_process.pop_back(); + if (node_to_process == table_expression) + return true; + + if (node_to_process->getNodeType() == QueryTreeNodeType::JOIN) + { + const auto & join_node = node_to_process->as(); + nodes_to_process.push_back(join_node.getLeftTableExpression()); + nodes_to_process.push_back(join_node.getRightTableExpression()); + } + } + return false; +} + +/// Columns that resolved from matcher can also match columns from JOIN USING. +/// In that case we update type to type of column in USING section. +/// TODO: It's not completely correct for qualified matchers, so t1.* should be resolved to left table column type. +/// But in planner we do not distinguish such cases. +void QueryAnalyzer::updateMatchedColumnsFromJoinUsing( + QueryTreeNodesWithNames & result_matched_column_nodes_with_names, + const QueryTreeNodePtr & source_table_expression, + IdentifierResolveScope & scope) +{ + auto * nearest_query_scope = scope.getNearestQueryScope(); + auto * nearest_query_scope_query_node = nearest_query_scope ? nearest_query_scope->scope_node->as() : nullptr; + + /// If there are no parent query scope or query scope does not have join tree + if (!nearest_query_scope_query_node || !nearest_query_scope_query_node->getJoinTree()) + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "There are no table sources. In scope {}", + scope.scope_node->formatASTForErrorMessage()); + } + + const auto & join_tree = nearest_query_scope_query_node->getJoinTree(); + + const auto * join_node = join_tree->as(); + if (join_node && join_node->isUsingJoinExpression()) + { + const auto & join_using_list = join_node->getJoinExpression()->as(); + const auto & join_using_nodes = join_using_list.getNodes(); + + for (auto & [matched_column_node, _] : result_matched_column_nodes_with_names) + { + auto & matched_column_node_typed = matched_column_node->as(); + const auto & matched_column_name = matched_column_node_typed.getColumnName(); + + for (const auto & join_using_node : join_using_nodes) + { + auto & join_using_column_node = join_using_node->as(); + const auto & join_using_column_name = join_using_column_node.getColumnName(); + + if (matched_column_name != join_using_column_name) + continue; + + const auto & join_using_column_nodes_list = join_using_column_node.getExpressionOrThrow()->as(); + const auto & join_using_column_nodes = join_using_column_nodes_list.getNodes(); + + auto it = node_to_projection_name.find(matched_column_node); + + if (hasTableExpressionInJoinTree(join_node->getLeftTableExpression(), source_table_expression)) + matched_column_node = join_using_column_nodes.at(0); + else if (hasTableExpressionInJoinTree(join_node->getRightTableExpression(), source_table_expression)) + matched_column_node = join_using_column_nodes.at(1); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot find column {} in JOIN USING section {}", + matched_column_node->dumpTree(), join_node->dumpTree()); + + matched_column_node = matched_column_node->clone(); + if (it != node_to_projection_name.end()) + node_to_projection_name.emplace(matched_column_node, it->second); + + matched_column_node->as().setColumnType(join_using_column_node.getResultType()); + } + } + } +} + /** Resolve qualified tree matcher. * * First try to match qualified identifier to expression. If qualified identifier matched expression node then @@ -4321,6 +4415,8 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveQualifiedMatcher(Qu matched_columns, scope); + updateMatchedColumnsFromJoinUsing(result_matched_column_nodes_with_names, table_expression_node, scope); + return result_matched_column_nodes_with_names; } diff --git a/tests/queries/0_stateless/00722_inner_join.reference b/tests/queries/0_stateless/00722_inner_join.reference index c7ba26e3e6a..b5e8a77a20d 100644 --- a/tests/queries/0_stateless/00722_inner_join.reference +++ b/tests/queries/0_stateless/00722_inner_join.reference @@ -31,9 +31,9 @@ ┌─database─┬─name─┐ │ system │ one │ └──────────┴──────┘ -┌─database─┬─t.name─┐ -│ system │ one │ -└──────────┴────────┘ +┌─database─┬─name─┐ +│ system │ one │ +└──────────┴──────┘ 2 2 2 diff --git a/tests/queries/0_stateless/02835_join_step_explain.reference b/tests/queries/0_stateless/02835_join_step_explain.reference index d2042b56f55..06f4a9cfc99 100644 --- a/tests/queries/0_stateless/02835_join_step_explain.reference +++ b/tests/queries/0_stateless/02835_join_step_explain.reference @@ -56,20 +56,20 @@ Header: id UInt64 rhs.id UInt64 rhs.value_1 String Actions: INPUT : 0 -> __table1.id UInt64 : 0 - INPUT :: 1 -> __table1.value_2 UInt64 : 1 - INPUT : 2 -> __table1.value_1 String : 2 + INPUT : 1 -> __table1.value_1 String : 1 + INPUT :: 2 -> __table1.value_2 UInt64 : 2 INPUT : 3 -> __table2.value_1 String : 3 INPUT :: 4 -> __table2.value_2 UInt64 : 4 INPUT : 5 -> __table2.id UInt64 : 5 ALIAS __table1.id :: 0 -> id UInt64 : 6 - ALIAS __table1.value_1 :: 2 -> value_1 String : 0 - ALIAS __table2.value_1 :: 3 -> rhs.value_1 String : 2 + ALIAS __table1.value_1 :: 1 -> value_1 String : 0 + ALIAS __table2.value_1 :: 3 -> rhs.value_1 String : 1 ALIAS __table2.id :: 5 -> rhs.id UInt64 : 3 -Positions: 6 0 3 2 +Positions: 6 0 3 1 Join (JOIN FillRightFirst) Header: __table1.id UInt64 - __table1.value_2 UInt64 __table1.value_1 String + __table1.value_2 UInt64 __table2.value_1 String __table2.value_2 UInt64 __table2.id UInt64 @@ -80,37 +80,37 @@ Positions: 6 0 3 2 Clauses: [(__table1.id, __table1.value_2) = (__table2.id, __table2.value_2)] Expression ((JOIN actions + Change column names to column identifiers)) Header: __table1.id UInt64 - __table1.value_2 UInt64 __table1.value_1 String + __table1.value_2 UInt64 Actions: INPUT : 0 -> id UInt64 : 0 - INPUT : 1 -> value_2 UInt64 : 1 - INPUT : 2 -> value_1 String : 2 + INPUT : 1 -> value_1 String : 1 + INPUT : 2 -> value_2 UInt64 : 2 ALIAS id :: 0 -> __table1.id UInt64 : 3 - ALIAS value_2 :: 1 -> __table1.value_2 UInt64 : 0 - ALIAS value_1 :: 2 -> __table1.value_1 String : 1 + ALIAS value_1 :: 1 -> __table1.value_1 String : 0 + ALIAS value_2 :: 2 -> __table1.value_2 UInt64 : 1 Positions: 3 0 1 ReadFromMergeTree (default.test_table_1) Header: id UInt64 - value_2 UInt64 value_1 String + value_2 UInt64 ReadType: Default Parts: 1 Granules: 1 Expression ((JOIN actions + Change column names to column identifiers)) Header: __table2.id UInt64 - __table2.value_2 UInt64 __table2.value_1 String + __table2.value_2 UInt64 Actions: INPUT : 0 -> id UInt64 : 0 - INPUT : 1 -> value_2 UInt64 : 1 - INPUT : 2 -> value_1 String : 2 + INPUT : 1 -> value_1 String : 1 + INPUT : 2 -> value_2 UInt64 : 2 ALIAS id :: 0 -> __table2.id UInt64 : 3 - ALIAS value_2 :: 1 -> __table2.value_2 UInt64 : 0 - ALIAS value_1 :: 2 -> __table2.value_1 String : 1 + ALIAS value_1 :: 1 -> __table2.value_1 String : 0 + ALIAS value_2 :: 2 -> __table2.value_2 UInt64 : 1 Positions: 3 0 1 ReadFromMergeTree (default.test_table_2) Header: id UInt64 - value_2 UInt64 value_1 String + value_2 UInt64 ReadType: Default Parts: 1 Granules: 1 diff --git a/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference b/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference.j2 similarity index 94% rename from tests/queries/0_stateless/02955_analyzer_using_functional_args.reference rename to tests/queries/0_stateless/02955_analyzer_using_functional_args.reference.j2 index f4d8a77ef7f..7d8fcf5bce0 100644 --- a/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference +++ b/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference.j2 @@ -1,3 +1,5 @@ +{% for column_expression_type in ['ALIAS', 'MATERIALIZED'] -%} +{{ column_expression_type }} 1 1 369 124 123 b @@ -6,7 +8,7 @@ 3693 1231 a 1231 3693 1232 1231 1231 a a --- {echoOn } +-- { echoOn } -- USING alias column contains default in old analyzer (but both queries below should have the same result) SELECT y * 2, s || 'a' FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 738 ba @@ -41,9 +43,9 @@ SELECT (1, t1.*, t2.*) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2 (1,'a',3693,0) (1,'b',369,369) SELECT t1.z, t2.z, t3.z FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1; -1 0 43 -1 48 1 -124 124 1 +0 0 43 +0 48 0 +124 124 0 1232 0 1232 SELECT * FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1; 126 0 0 42 @@ -79,3 +81,4 @@ b DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; +{% endfor -%} diff --git a/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql b/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql.j2 similarity index 86% rename from tests/queries/0_stateless/02955_analyzer_using_functional_args.sql rename to tests/queries/0_stateless/02955_analyzer_using_functional_args.sql.j2 index f92598c7c32..6fb88a02e5e 100644 --- a/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql +++ b/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql.j2 @@ -2,10 +2,16 @@ DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; -CREATE TABLE t1 (x Int16, y Int64 ALIAS x + x * 2, z ALIAS x + 1, s String) ENGINE=MergeTree() ORDER BY x; +{% for column_expression_type in ['ALIAS', 'MATERIALIZED'] %} + +-- { echoOff } + +SELECT '{{ column_expression_type }}'; + +CREATE TABLE t1 (x Int16, y Int64 {{ column_expression_type }} x + x * 2, z {{ column_expression_type }} x + 1, s String) ENGINE=MergeTree() ORDER BY x; CREATE TABLE t2 (y Int128, z Int16) ENGINE=MergeTree() ORDER BY y; -CREATE TABLE t3 (x Int16, y Int64 ALIAS x + x * 2, z ALIAS x + 1) ENGINE=MergeTree() ORDER BY x; +CREATE TABLE t3 (x Int16, y Int64 {{ column_expression_type }} x + x * 2, z {{ column_expression_type }} x + 1) ENGINE=MergeTree() ORDER BY x; INSERT INTO t1 VALUES (1231, 'a'), (123, 'b'); @@ -26,7 +32,7 @@ SELECT * FROM t1 INNER JOIN t3 USING (y) SETTINGS allow_experimental_analyzer = SELECT * FROM t3 INNER JOIN t1 USING (y, z) SETTINGS allow_experimental_analyzer = 1; SELECT s FROM t1 INNER JOIN t3 USING (y); --- {echoOn } +-- { echoOn } -- USING alias column contains default in old analyzer (but both queries below should have the same result) SELECT y * 2, s || 'a' FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; SELECT y * 2, s || 'a' FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; @@ -51,7 +57,8 @@ SELECT y FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) SELECT s FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL; SELECT s FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL; - DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; + +{% endfor %} From 4b144f94f8ef2f3fa3cc90fcf6431f4d75691189 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 4 Mar 2024 14:24:55 +0300 Subject: [PATCH 0328/1165] MergeTree read split ranges into intersecting and non intersecting injection --- src/Core/Settings.h | 1 + src/Core/SettingsChangesHistory.h | 1 + src/Processors/QueryPlan/PartsSplitter.cpp | 10 ++- .../QueryPlan/ReadFromMergeTree.cpp | 76 +++++++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5c25368fb00..b7ed8fc7d1d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -287,6 +287,7 @@ class IColumn; M(UInt64, read_backoff_min_concurrency, 1, "Settings to try keeping the minimal number of threads in case of slow reads.", 0) \ \ M(Float, memory_tracker_fault_probability, 0., "For testing of `exception safety` - throw an exception every time you allocate memory with the specified probability.", 0) \ + M(Float, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability, 1.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability.", 0) \ \ M(Bool, enable_http_compression, false, "Compress the result if the client over HTTP said that it understands data compressed by gzip, deflate, zstd, br, lz4, bz2, xz.", 0) \ M(Int64, http_zlib_compression_level, 3, "Compression level - used if the client on HTTP said that it understands data compressed by gzip or deflate.", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 0bce814170b..e8e38ca82f9 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -131,6 +131,7 @@ static std::map sett {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, + {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, }}, {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 5235785907c..33151597a38 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -34,6 +34,12 @@ std::string toString(const Values & value) return fmt::format("({})", fmt::join(value, ", ")); } +/** We rely that FieldVisitorAccurateLess will have strict weak ordering for any Field values including + * NaN, Null and containers (Array, Tuple, Map) that contain NaN or Null. But right now it does not properly + * support NaN and Nulls inside containers, because it uses Field operator< or accurate::lessOp for comparison + * that compares Nulls and NaNs differently than FieldVisitorAccurateLess. + * TODO: Update Field operator< to compare NaNs and Nulls the same way as FieldVisitorAccurateLess. + */ bool isSafePrimaryDataKeyType(const IDataType & data_type) { auto type_id = data_type.getTypeId(); @@ -316,12 +322,12 @@ struct PartRangeIndex bool operator==(const PartRangeIndex & other) const { - return part_index == other.part_index && range.begin == other.range.begin && range.end == other.range.end; + return std::tie(part_index, range.begin, range.end) == std::tie(other.part_index, other.range.begin, other.range.end); } bool operator<(const PartRangeIndex & other) const { - return part_index < other.part_index && range.begin < other.range.begin && range.end < other.range.end; + return std::tie(part_index, range.begin, range.end) < std::tie(other.part_index, other.range.begin, other.range.end); } size_t part_index; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index f1ce5a7802f..4689e67a1ae 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -766,6 +766,82 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_ auto read_type = is_parallel_reading_from_replicas ? ReadType::ParallelReplicas : ReadType::Default; + double read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = settings.merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability; + std::bernoulli_distribution fault(read_split_ranges_into_intersecting_and_non_intersecting_injection_probability); + + if (read_type != ReadType::ParallelReplicas && + num_streams > 1 && + read_split_ranges_into_intersecting_and_non_intersecting_injection_probability >= 0.0 && + fault(thread_local_rng) && + !isQueryWithFinal() && + data.merging_params.is_deleted_column.empty() && + !prewhere_info) + { + NameSet column_names_set(column_names.begin(), column_names.end()); + Names in_order_column_names_to_read(column_names); + + /// Add columns needed to calculate the sorting expression + for (const auto & column_name : metadata_for_reading->getColumnsRequiredForSortingKey()) + { + if (column_names_set.contains(column_name)) + continue; + + in_order_column_names_to_read.push_back(column_name); + column_names_set.insert(column_name); + } + + auto in_order_reading_step_getter = [this, &in_order_column_names_to_read, &info](auto parts) + { + return this->read( + std::move(parts), + in_order_column_names_to_read, + ReadType::InOrder, + 1 /* num_streams */, + 0 /* min_marks_for_concurrent_read */, + info.use_uncompressed_cache); + }; + + auto sorting_expr = std::make_shared(metadata_for_reading->getSortingKey().expression->getActionsDAG().clone()); + + SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey( + metadata_for_reading->getPrimaryKey(), + std::move(sorting_expr), + std::move(parts_with_ranges), + num_streams, + context, + std::move(in_order_reading_step_getter), + true /*split_parts_ranges_into_intersecting_and_non_intersecting_final*/, + true /*split_intersecting_parts_ranges_into_layers*/); + + auto merging_pipes = std::move(split_ranges_result.merging_pipes); + auto non_intersecting_parts_ranges_read_pipe = read(std::move(split_ranges_result.non_intersecting_parts_ranges), + column_names, + read_type, + num_streams, + info.min_marks_for_concurrent_read, + info.use_uncompressed_cache); + + if (merging_pipes.empty()) + return non_intersecting_parts_ranges_read_pipe; + + Pipes pipes; + pipes.resize(2); + pipes[0] = Pipe::unitePipes(std::move(merging_pipes)); + pipes[1] = std::move(non_intersecting_parts_ranges_read_pipe); + + auto conversion_action = ActionsDAG::makeConvertingActions( + pipes[0].getHeader().getColumnsWithTypeAndName(), + pipes[1].getHeader().getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + pipes[0].addSimpleTransform( + [conversion_action](const Block & header) + { + auto converting_expr = std::make_shared(conversion_action); + return std::make_shared(header, converting_expr); + }); + return Pipe::unitePipes(std::move(pipes)); + } + return read(std::move(parts_with_ranges), column_names, read_type, From a1435fc915a151ef057065854f8b8ed79fd7af2c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 4 Mar 2024 14:27:12 +0300 Subject: [PATCH 0329/1165] Fixed tests --- .../00278_insert_already_sorted.sql | 4 +- .../0_stateless/00319_index_for_like.sql | 4 +- .../0_stateless/00612_pk_in_tuple_perf.sh | 4 +- .../0_stateless/00943_materialize_index.sh | 20 +++--- .../00944_clear_index_in_partition.sh | 12 ++-- .../0_stateless/00945_bloom_filter_index.sql | 1 + .../00974_primary_key_for_lowCardinality.sh | 4 +- ...ices_mutation_replicated_zookeeper_long.sh | 12 ++-- .../00979_toFloat_monotonicity.sql | 2 + ..._materialize_clear_index_compact_parts.sql | 2 + .../01200_mutations_memory_consumption.sql | 2 + .../0_stateless/01234_to_string_monotonic.sql | 2 + ...with_constant_string_in_index_analysis.sql | 2 + .../01585_use_index_for_global_in.sql | 2 + .../0_stateless/01624_soft_constraints.sh | 2 +- .../01710_aggregate_projections.sh | 2 +- .../0_stateless/01710_projection_in_index.sql | 2 + .../01748_partition_id_pruning.sql | 2 + .../02155_read_in_order_max_rows_to_read.sql | 2 + ...3_optimize_aggregation_in_order_prefix.sql | 2 + .../02340_parts_refcnt_mergetree.sh | 4 +- .../02343_aggregation_pipeline.sql | 1 + .../02346_inverted_index_search.sql | 61 ++++++++++--------- ...ting_by_input_stream_properties_explain.sh | 6 +- .../02402_merge_engine_with_view.sql | 2 + .../02418_tautological_if_index.sql | 2 + ...493_max_streams_for_merge_tree_reading.sql | 2 + .../02494_query_cache_nested_query_bug.sh | 8 ++- .../02521_aggregation_by_partitions.sql | 2 + .../0_stateless/02532_send_logs_level_test.sh | 2 +- .../02582_async_reading_with_small_limit.sql | 2 + .../02714_read_bytes_aggregateFunction.sql | 2 + .../02862_sorted_distinct_sparse_fix.sql | 2 + ...02899_indexing_by_space_filling_curves.sql | 2 + .../0_stateless/02949_ttl_group_by_bug.sql | 2 + .../02950_part_offset_as_primary_key.sql | 2 + 36 files changed, 118 insertions(+), 69 deletions(-) diff --git a/tests/queries/0_stateless/00278_insert_already_sorted.sql b/tests/queries/0_stateless/00278_insert_already_sorted.sql index b3de48dc155..dbd129d85cd 100644 --- a/tests/queries/0_stateless/00278_insert_already_sorted.sql +++ b/tests/queries/0_stateless/00278_insert_already_sorted.sql @@ -7,11 +7,11 @@ INSERT INTO sorted (x) SELECT intDiv(number, 100000) AS x FROM system.numbers LI SET max_threads = 1; SELECT count() FROM sorted; -SELECT DISTINCT x FROM sorted; +SELECT x FROM (SELECT DISTINCT x FROM sorted) ORDER BY x; INSERT INTO sorted (x) SELECT (intHash64(number) % 1000 = 0 ? 999 : intDiv(number, 100000)) AS x FROM system.numbers LIMIT 1000000; SELECT count() FROM sorted; -SELECT DISTINCT x FROM sorted; +SELECT x FROM (SELECT DISTINCT x FROM sorted) ORDER BY x; DROP TABLE sorted; diff --git a/tests/queries/0_stateless/00319_index_for_like.sql b/tests/queries/0_stateless/00319_index_for_like.sql index e490e595142..6abd1edc3ed 100644 --- a/tests/queries/0_stateless/00319_index_for_like.sql +++ b/tests/queries/0_stateless/00319_index_for_like.sql @@ -1,8 +1,10 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS index_for_like; set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE index_for_like (s String, d Date DEFAULT today()) ENGINE = MergeTree(d, (s, d), 1); -INSERT INTO index_for_like (s) VALUES ('Hello'), ('Hello, World'), ('Hello, World 1'), ('Hello 1'), ('Goodbye'), ('Goodbye, World'), ('Goodbye 1'), ('Goodbye, World 1'); +INSERT INTO index_for_like (s) VALUES ('Hello'), ('Hello, World'), ('Hello, World 1'), ('Hello 1'), ('Goodbye'), ('Goodbye, World'), ('Goodbye 1'), ('Goodbye, World 1'); SET max_rows_to_read = 3; SELECT s FROM index_for_like WHERE s LIKE 'Hello, World%'; diff --git a/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh b/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh index 99813d894ae..c8297635c43 100755 --- a/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh +++ b/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh @@ -20,7 +20,7 @@ SETTINGS index_granularity = 1; INSERT INTO pk_in_tuple_perf SELECT number, number * 10 FROM numbers(100); EOF -query="SELECT count() FROM pk_in_tuple_perf WHERE (v, u) IN ((2, 10), (2, 20))" +query="SELECT count() FROM pk_in_tuple_perf WHERE (v, u) IN ((2, 10), (2, 20)) SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0" $CLICKHOUSE_CLIENT --query "$query" $CLICKHOUSE_CLIENT --query "$query FORMAT JSON" | grep "rows_read" @@ -40,7 +40,7 @@ SETTINGS index_granularity = 1; INSERT INTO pk_in_tuple_perf_non_const SELECT today() - number, number FROM numbers(100); EOF -query="SELECT count() FROM pk_in_tuple_perf_non_const WHERE (u, d) IN ((0, today()), (1, today()))" +query="SELECT count() FROM pk_in_tuple_perf_non_const WHERE (u, d) IN ((0, today()), (1, today())) SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0" $CLICKHOUSE_CLIENT --query "$query" $CLICKHOUSE_CLIENT --query "$query FORMAT JSON" | grep "rows_read" diff --git a/tests/queries/0_stateless/00943_materialize_index.sh b/tests/queries/0_stateless/00943_materialize_index.sh index 30ef46e5cb0..6ff7d34a9d7 100755 --- a/tests/queries/0_stateless/00943_materialize_index.sh +++ b/tests/queries/0_stateless/00943_materialize_index.sh @@ -31,31 +31,31 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES (8, 1, 2), (9, 1, 2)" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT -n --query=" ALTER TABLE minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1 SETTINGS mutations_sync = 2;" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 1 SETTINGS mutations_sync = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 2 SETTINGS mutations_sync = 2" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx CLEAR INDEX idx IN PARTITION 1 SETTINGS mutations_sync = 2" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx CLEAR INDEX idx IN PARTITION 2 SETTINGS mutations_sync = 2" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx SETTINGS mutations_sync = 2" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx" diff --git a/tests/queries/0_stateless/00944_clear_index_in_partition.sh b/tests/queries/0_stateless/00944_clear_index_in_partition.sh index 8b74bd94f2c..4655077960f 100755 --- a/tests/queries/0_stateless/00944_clear_index_in_partition.sh +++ b/tests/queries/0_stateless/00944_clear_index_in_partition.sh @@ -32,17 +32,17 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES (8, 1, 2), (9, 1, 2)" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" # Returns 4 +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" # Returns 4 $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx CLEAR INDEX idx IN PARTITION 1;" --mutations_sync=1 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" # Returns 6 +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" # Returns 6 $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" --mutations_sync=1 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" # Returns 4 +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" # Returns 4 $CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx" diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.sql b/tests/queries/0_stateless/00945_bloom_filter_index.sql index faa7feda04d..4c26988574a 100644 --- a/tests/queries/0_stateless/00945_bloom_filter_index.sql +++ b/tests/queries/0_stateless/00945_bloom_filter_index.sql @@ -1,4 +1,5 @@ SET allow_suspicious_low_cardinality_types=1; +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; DROP TABLE IF EXISTS single_column_bloom_filter; diff --git a/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh b/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh index f8527cd491e..389d433c7e2 100755 --- a/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh +++ b/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh @@ -32,9 +32,9 @@ $CLICKHOUSE_CLIENT --query="insert into lowString (a, b) select top 100000 toStr $CLICKHOUSE_CLIENT --query="insert into string (a, b) select top 100000 toString(number), today() from system.numbers" -$CLICKHOUSE_CLIENT --query="select count() from lowString where a in ('1', '2') FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="select count() from lowString where a in ('1', '2') SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="select count() from string where a in ('1', '2') FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="select count() from string where a in ('1', '2') SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="DROP TABLE lowString;" $CLICKHOUSE_CLIENT --query="DROP TABLE string;" diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh index 89b17ffe2bf..686dd7f6df0 100755 --- a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh +++ b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh @@ -49,18 +49,18 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO indices_mutaions1 VALUES $CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA indices_mutaions2" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="DROP TABLE indices_mutaions1" $CLICKHOUSE_CLIENT --query="DROP TABLE indices_mutaions2" diff --git a/tests/queries/0_stateless/00979_toFloat_monotonicity.sql b/tests/queries/0_stateless/00979_toFloat_monotonicity.sql index 9b3bdf91573..fa1f5b17a1c 100644 --- a/tests/queries/0_stateless/00979_toFloat_monotonicity.sql +++ b/tests/queries/0_stateless/00979_toFloat_monotonicity.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test1; DROP TABLE IF EXISTS test2; DROP TABLE IF EXISTS test3; diff --git a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql index 767ca0e4073..b2ebe7e2cc2 100644 --- a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql +++ b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS minmax_compact; CREATE TABLE minmax_compact diff --git a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql index bca2286aa22..61263d125b0 100644 --- a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql +++ b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql @@ -1,5 +1,7 @@ -- Tags: no-debug, no-parallel, long, no-s3-storage, no-random-merge-tree-settings +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS table_with_single_pk; CREATE TABLE table_with_single_pk diff --git a/tests/queries/0_stateless/01234_to_string_monotonic.sql b/tests/queries/0_stateless/01234_to_string_monotonic.sql index 87324fdda27..65b7ab21073 100644 --- a/tests/queries/0_stateless/01234_to_string_monotonic.sql +++ b/tests/queries/0_stateless/01234_to_string_monotonic.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test1; DROP TABLE IF EXISTS test2; diff --git a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql index 50aa434a28c..b7778dfd780 100644 --- a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql +++ b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test; CREATE TABLE test (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 1000, index_granularity_bytes = '10Mi'; INSERT INTO test SELECT * FROM numbers(1000000); diff --git a/tests/queries/0_stateless/01585_use_index_for_global_in.sql b/tests/queries/0_stateless/01585_use_index_for_global_in.sql index 1dd7609350f..3b0ca726dfa 100644 --- a/tests/queries/0_stateless/01585_use_index_for_global_in.sql +++ b/tests/queries/0_stateless/01585_use_index_for_global_in.sql @@ -1,5 +1,7 @@ -- Tags: global +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists xp; drop table if exists xp_d; diff --git a/tests/queries/0_stateless/01624_soft_constraints.sh b/tests/queries/0_stateless/01624_soft_constraints.sh index 944a4e4234f..aae193f6a5d 100755 --- a/tests/queries/0_stateless/01624_soft_constraints.sh +++ b/tests/queries/0_stateless/01624_soft_constraints.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -SETTINGS="SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1; SET optimize_substitute_columns = 1; SET optimize_append_index = 1" +SETTINGS="SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1; SET optimize_substitute_columns = 1; SET optimize_append_index = 1; SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" $CLICKHOUSE_CLIENT -n --query=" $SETTINGS; diff --git a/tests/queries/0_stateless/01710_aggregate_projections.sh b/tests/queries/0_stateless/01710_aggregate_projections.sh index 7ea40365937..b02d961bf97 100755 --- a/tests/queries/0_stateless/01710_aggregate_projections.sh +++ b/tests/queries/0_stateless/01710_aggregate_projections.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh # Number of read rows depends on max_bytes_before_external_group_by. -CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0" +CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability 0.0" $CLICKHOUSE_CLIENT -q "CREATE TABLE test_agg_proj (x Int32, y Int32, PROJECTION x_plus_y (SELECT sum(x - y), argMax(x, y) group by x + y)) ENGINE = MergeTree ORDER BY tuple() settings index_granularity = 1" $CLICKHOUSE_CLIENT -q "insert into test_agg_proj select intDiv(number, 2), -intDiv(number,3) - 1 from numbers(100)" diff --git a/tests/queries/0_stateless/01710_projection_in_index.sql b/tests/queries/0_stateless/01710_projection_in_index.sql index b858418584c..425653ec501 100644 --- a/tests/queries/0_stateless/01710_projection_in_index.sql +++ b/tests/queries/0_stateless/01710_projection_in_index.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists t; create table t (i int, j int, k int, projection p (select * order by j)) engine MergeTree order by i settings index_granularity = 1; diff --git a/tests/queries/0_stateless/01748_partition_id_pruning.sql b/tests/queries/0_stateless/01748_partition_id_pruning.sql index 9a26dd8daba..b637528bc6c 100644 --- a/tests/queries/0_stateless/01748_partition_id_pruning.sql +++ b/tests/queries/0_stateless/01748_partition_id_pruning.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists x; create table x (i int, j int) engine MergeTree partition by i order by j settings index_granularity = 1; diff --git a/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql index b387582296d..4b47a860071 100644 --- a/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql +++ b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS t_max_rows_to_read; CREATE TABLE t_max_rows_to_read (a UInt64) diff --git a/tests/queries/0_stateless/02233_optimize_aggregation_in_order_prefix.sql b/tests/queries/0_stateless/02233_optimize_aggregation_in_order_prefix.sql index 5065bd96bc1..8bc75040e5a 100644 --- a/tests/queries/0_stateless/02233_optimize_aggregation_in_order_prefix.sql +++ b/tests/queries/0_stateless/02233_optimize_aggregation_in_order_prefix.sql @@ -1,5 +1,7 @@ -- Tags: no-s3-storage +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists data_02233; create table data_02233 (parent_key Int, child_key Int, value Int) engine=MergeTree() order by parent_key; diff --git a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh index 9ea924377b2..208a9038681 100755 --- a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh +++ b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh @@ -15,11 +15,13 @@ function check_refcnt_for_table() local query_id query_id="$table-$(random_str 10)" + SETTINGS="--format Null --max_threads 1 --max_block_size 1 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability 0.0" + # Notes: # - query may sleep 1*(200/4)=50 seconds maximum, it is enough to check system.parts # - "part = 1" condition should prune all parts except first # - max_block_size=1 with index_granularity=1 will allow to cancel the query earlier - $CLICKHOUSE_CLIENT --format Null --max_threads 1 --max_block_size 1 --query_id "$query_id" -q "select sleepEachRow(1) from $table where part = 1" & + $CLICKHOUSE_CLIENT $SETTINGS --query_id "$query_id" -q "select sleepEachRow(1) from $table where part = 1" & PID=$! # wait for query to be started diff --git a/tests/queries/0_stateless/02343_aggregation_pipeline.sql b/tests/queries/0_stateless/02343_aggregation_pipeline.sql index 496379f5c48..d73ac66763e 100644 --- a/tests/queries/0_stateless/02343_aggregation_pipeline.sql +++ b/tests/queries/0_stateless/02343_aggregation_pipeline.sql @@ -2,6 +2,7 @@ -- produces different pipeline if enabled set enable_memory_bound_merging_of_aggregation_results = 0; +set merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; set max_threads = 16; set prefer_localhost_replica = 1; diff --git a/tests/queries/0_stateless/02346_inverted_index_search.sql b/tests/queries/0_stateless/02346_inverted_index_search.sql index d225d3463d1..052703dceaf 100644 --- a/tests/queries/0_stateless/02346_inverted_index_search.sql +++ b/tests/queries/0_stateless/02346_inverted_index_search.sql @@ -1,5 +1,6 @@ SET allow_experimental_inverted_index = 1; SET log_queries = 1; +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; ---------------------------------------------------- SELECT 'Test inverted(2)'; @@ -23,7 +24,7 @@ SELECT * FROM tab WHERE s == 'Alick a01'; -- check the query only read 1 granules (2 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==2 from system.query_log +SELECT read_rows==2 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s == \'Alick a01\';') @@ -36,7 +37,7 @@ SELECT * FROM tab WHERE s LIKE '%01%' ORDER BY k; -- check the query only read 2 granules (4 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==4 from system.query_log +SELECT read_rows==4 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s LIKE \'%01%\' ORDER BY k;') @@ -49,11 +50,11 @@ SELECT * FROM tab WHERE hasToken(s, 'Click') ORDER BY k; -- check the query only read 4 granules (8 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==8 from system.query_log +SELECT read_rows==8 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE hasToken(s, \'Click\') ORDER BY k;') - AND type='QueryFinish' + AND type='QueryFinish' AND result_rows==4 LIMIT 1; @@ -76,11 +77,11 @@ SELECT * FROM tab_x WHERE hasToken(s, 'Alick') ORDER BY k; -- check the query only read 4 granules (8 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==8 from system.query_log +SELECT read_rows==8 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE hasToken(s, \'Alick\');') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE hasToken(s, \'Alick\');') + AND type='QueryFinish' AND result_rows==4 LIMIT 1; @@ -89,24 +90,24 @@ SELECT * FROM tab_x WHERE s IN ('Alick a01', 'Alick a06') ORDER BY k; -- check the query only read 2 granules (4 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==4 from system.query_log +SELECT read_rows==4 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE s IN (\'Alick a01\', \'Alick a06\') ORDER BY k;') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE s IN (\'Alick a01\', \'Alick a06\') ORDER BY k;') + AND type='QueryFinish' AND result_rows==2 LIMIT 1; --- search inverted index with multiSearch +-- search inverted index with multiSearch SELECT * FROM tab_x WHERE multiSearchAny(s, ['a01', 'b01']) ORDER BY k; -- check the query only read 2 granules (4 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==4 from system.query_log +SELECT read_rows==4 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE multiSearchAny(s, [\'a01\', \'b01\']) ORDER BY k;') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE multiSearchAny(s, [\'a01\', \'b01\']) ORDER BY k;') + AND type='QueryFinish' AND result_rows==2 LIMIT 1; @@ -129,11 +130,11 @@ SELECT * FROM tab WHERE has(s, 'Click a03') ORDER BY k; -- check the query must read all 10 granules (20 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==2 from system.query_log +SELECT read_rows==2 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE has(s, \'Click a03\') ORDER BY k;') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE has(s, \'Click a03\') ORDER BY k;') + AND type='QueryFinish' AND result_rows==1 LIMIT 1; @@ -156,11 +157,11 @@ SELECT * FROM tab WHERE mapContains(s, 'Click') ORDER BY k; -- check the query must read all 4 granules (8 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==8 from system.query_log +SELECT read_rows==8 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE mapContains(s, \'Click\') ORDER BY k;') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE mapContains(s, \'Click\') ORDER BY k;') + AND type='QueryFinish' AND result_rows==4 LIMIT 1; @@ -169,11 +170,11 @@ SELECT * FROM tab WHERE s['Click'] = 'Click a03'; -- check the query must read all 4 granules (8 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==8 from system.query_log +SELECT read_rows==8 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s[\'Click\'] = \'Click a03\';') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s[\'Click\'] = \'Click a03\';') + AND type='QueryFinish' AND result_rows==1 LIMIT 1; @@ -199,10 +200,10 @@ SELECT * FROM tab WHERE s LIKE '%01%' ORDER BY k; -- check the query only read 3 granules (6 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; SELECT read_rows==6 from system.query_log - WHERE query_kind ='Select' + WHERE query_kind ='Select' AND current_database = currentDatabase() AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s LIKE \'%01%\' ORDER BY k;') - AND type='QueryFinish' + AND type='QueryFinish' AND result_rows==3 LIMIT 1; @@ -226,11 +227,11 @@ SELECT * FROM tab WHERE s LIKE '%你好%' ORDER BY k; -- check the query only read 1 granule (2 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==2 from system.query_log - WHERE query_kind ='Select' - AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s LIKE \'%你好%\' ORDER BY k;') - AND type='QueryFinish' +SELECT read_rows==2 from system.query_log + WHERE query_kind ='Select' + AND current_database = currentDatabase() + AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s LIKE \'%你好%\' ORDER BY k;') + AND type='QueryFinish' AND result_rows==1 LIMIT 1; diff --git a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh index 7e937ac42b6..c223fcc86bc 100755 --- a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh +++ b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh @@ -15,14 +15,14 @@ FIND_SORTMODE="$GREP_SORTMODE | $TRIM_LEADING_SPACES" function explain_sorting { echo "-- QUERY: "$1 - $CLICKHOUSE_CLIENT -nq "$1" | eval $FIND_SORTING + $CLICKHOUSE_CLIENT --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTING } function explain_sortmode { echo "-- QUERY: "$1 - $CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -nq "$1" | eval $FIND_SORTMODE + $CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTMODE echo "-- QUERY (analyzer): "$1 - $CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -nq "$1" | eval $FIND_SORTMODE + $CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTMODE } $CLICKHOUSE_CLIENT -q "drop table if exists optimize_sorting sync" diff --git a/tests/queries/0_stateless/02402_merge_engine_with_view.sql b/tests/queries/0_stateless/02402_merge_engine_with_view.sql index 81c2d67d05b..3998f410a63 100644 --- a/tests/queries/0_stateless/02402_merge_engine_with_view.sql +++ b/tests/queries/0_stateless/02402_merge_engine_with_view.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + -- #40014 CREATE TABLE m0 (id UInt64) ENGINE=MergeTree ORDER BY id SETTINGS index_granularity = 1, ratio_of_defaults_for_sparse_serialization = 1.0; INSERT INTO m0 SELECT number FROM numbers(10); diff --git a/tests/queries/0_stateless/02418_tautological_if_index.sql b/tests/queries/0_stateless/02418_tautological_if_index.sql index c37f24b29f6..1368347e88f 100644 --- a/tests/queries/0_stateless/02418_tautological_if_index.sql +++ b/tests/queries/0_stateless/02418_tautological_if_index.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS constCondOptimization; CREATE TABLE constCondOptimization diff --git a/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql index f2e81273f12..cbf645ddec5 100644 --- a/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql +++ b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql @@ -1,5 +1,7 @@ -- Tags: no-random-merge-tree-settings +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists t; create table t (x UInt64) engine = MergeTree order by x; insert into t select number from numbers_mt(10000000) settings max_insert_threads=8; diff --git a/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh b/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh index 394367919e9..8712c7c84c6 100755 --- a/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh +++ b/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh @@ -15,9 +15,11 @@ ${CLICKHOUSE_CLIENT} --query "CREATE TABLE tab (a UInt64) ENGINE=MergeTree() ORD ${CLICKHOUSE_CLIENT} --query "INSERT INTO tab VALUES (1) (2) (3)" ${CLICKHOUSE_CLIENT} --query "INSERT INTO tab VALUES (3) (4) (5)" +SETTINGS="SETTINGS use_query_cache=1, max_threads=1, allow_experimental_analyzer=0, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0" + # Verify that the first query does two aggregations and the second query zero aggregations. Since query cache is currently not integrated -# with EXPLAIN PLAN, we need need to check the logs. -${CLICKHOUSE_CLIENT} --send_logs_level=trace --query "SELECT count(a) / (SELECT sum(a) FROM tab) FROM tab SETTINGS use_query_cache=1, max_threads=1, allow_experimental_analyzer=0" 2>&1 | grep "Aggregated. " | wc -l -${CLICKHOUSE_CLIENT} --send_logs_level=trace --query "SELECT count(a) / (SELECT sum(a) FROM tab) FROM tab SETTINGS use_query_cache=1, max_threads=1, allow_experimental_analyzer=0" 2>&1 | grep "Aggregated. " | wc -l +# with EXPLAIN PLAN, we need to check the logs. +${CLICKHOUSE_CLIENT} --send_logs_level=trace --query "SELECT count(a) / (SELECT sum(a) FROM tab) FROM tab $SETTINGS" 2>&1 | grep "Aggregated. " | wc -l +${CLICKHOUSE_CLIENT} --send_logs_level=trace --query "SELECT count(a) / (SELECT sum(a) FROM tab) FROM tab $SETTINGS" 2>&1 | grep "Aggregated. " | wc -l ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP QUERY CACHE" diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 5b013ca5aef..a90e56a9822 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -1,5 +1,7 @@ -- Tags: long, no-s3-storage +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + set max_threads = 16; set allow_aggregate_partitions_independently = 1; set force_aggregate_partitions_independently = 1; diff --git a/tests/queries/0_stateless/02532_send_logs_level_test.sh b/tests/queries/0_stateless/02532_send_logs_level_test.sh index 7dd9a152385..f65d8705569 100755 --- a/tests/queries/0_stateless/02532_send_logs_level_test.sh +++ b/tests/queries/0_stateless/02532_send_logs_level_test.sh @@ -17,6 +17,6 @@ $CLICKHOUSE_CLIENT -nm -q " # instead of "last" value, hence you cannot simply append another # --send_logs_level here. CLICKHOUSE_CLIENT_CLEAN=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=test/g') -$CLICKHOUSE_CLIENT_CLEAN -q "select * from data" |& grep -o -e '.*' -e '.*' +$CLICKHOUSE_CLIENT_CLEAN -q "select * from data SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" |& grep -o -e '.*' -e '.*' $CLICKHOUSE_CLIENT -q "drop table data" diff --git a/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql b/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql index e51666673b1..cb6b1b6083e 100644 --- a/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql +++ b/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql @@ -1,5 +1,7 @@ -- Tags: no-s3-storage +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists t; create table t(a UInt64) engine=MergeTree order by tuple(); diff --git a/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql b/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql index 1c70a77c4d1..eb0a37045ef 100644 --- a/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql +++ b/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + CREATE TABLE test (id UInt64, `amax` AggregateFunction(argMax, String, DateTime)) ENGINE=MergeTree() ORDER BY id diff --git a/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.sql b/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.sql index 2bcdb3d43ff..d303a53fc39 100644 --- a/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.sql +++ b/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS t_sparse_distinct; CREATE TABLE t_sparse_distinct (id UInt32, v String) diff --git a/tests/queries/0_stateless/02899_indexing_by_space_filling_curves.sql b/tests/queries/0_stateless/02899_indexing_by_space_filling_curves.sql index c7325b2478d..a3989039f50 100644 --- a/tests/queries/0_stateless/02899_indexing_by_space_filling_curves.sql +++ b/tests/queries/0_stateless/02899_indexing_by_space_filling_curves.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test; CREATE TABLE test (x UInt32, y UInt32) ENGINE = MergeTree ORDER BY mortonEncode(x, y) SETTINGS index_granularity = 8192, index_granularity_bytes = '1Mi'; diff --git a/tests/queries/0_stateless/02949_ttl_group_by_bug.sql b/tests/queries/0_stateless/02949_ttl_group_by_bug.sql index a3d0794c897..83776cc9d85 100644 --- a/tests/queries/0_stateless/02949_ttl_group_by_bug.sql +++ b/tests/queries/0_stateless/02949_ttl_group_by_bug.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS ttl_group_by_bug; CREATE TABLE ttl_group_by_bug diff --git a/tests/queries/0_stateless/02950_part_offset_as_primary_key.sql b/tests/queries/0_stateless/02950_part_offset_as_primary_key.sql index 736d54023ce..31a82cce1cd 100644 --- a/tests/queries/0_stateless/02950_part_offset_as_primary_key.sql +++ b/tests/queries/0_stateless/02950_part_offset_as_primary_key.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists a; create table a (i int) engine MergeTree order by i settings index_granularity = 2; From b8c53d7cf3f8de58ca8778be4bb87df4faaa6cca Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 6 Mar 2024 12:06:19 +0300 Subject: [PATCH 0330/1165] Fixed tests --- src/Core/Settings.h | 2 +- src/Processors/QueryPlan/PartsSplitter.cpp | 2 +- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- tests/ci/stress.py | 3 +++ tests/clickhouse-test | 1 + .../0_stateless/01200_mutations_memory_consumption.sql | 4 +--- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b7ed8fc7d1d..891bd8c0de9 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -287,7 +287,7 @@ class IColumn; M(UInt64, read_backoff_min_concurrency, 1, "Settings to try keeping the minimal number of threads in case of slow reads.", 0) \ \ M(Float, memory_tracker_fault_probability, 0., "For testing of `exception safety` - throw an exception every time you allocate memory with the specified probability.", 0) \ - M(Float, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability, 1.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability.", 0) \ + M(Float, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability.", 0) \ \ M(Bool, enable_http_compression, false, "Compress the result if the client over HTTP said that it understands data compressed by gzip, deflate, zstd, br, lz4, bz2, xz.", 0) \ M(Int64, http_zlib_compression_level, 3, "Compression level - used if the client on HTTP said that it understands data compressed by gzip or deflate.", 0) \ diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 33151597a38..2af1bcb0260 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -792,7 +792,7 @@ ASTs buildFilters(const KeyDescription & primary_key, const std::vector const auto & type = primary_key.data_types.at(i); // PK may contain functions of the table columns, so we need the actual PK AST with all expressions it contains. - auto pk_ast = primary_key.expression_list_ast->children.at(i); + auto pk_ast = primary_key.expression_list_ast->children.at(i)->clone(); // If PK is nullable, prepend a null mask column for > comparison. // Also transform the AST into assumeNotNull(pk) so that the result type is not-nullable. diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 4689e67a1ae..f4607cad040 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -771,7 +771,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_ if (read_type != ReadType::ParallelReplicas && num_streams > 1 && - read_split_ranges_into_intersecting_and_non_intersecting_injection_probability >= 0.0 && + read_split_ranges_into_intersecting_and_non_intersecting_injection_probability > 0.0 && fault(thread_local_rng) && !isQueryWithFinal() && data.merging_params.is_deleted_column.empty() && diff --git a/tests/ci/stress.py b/tests/ci/stress.py index 7ccc058f79f..dbc04835fae 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -52,6 +52,9 @@ def get_options(i: int, upgrade_check: bool) -> str: if i % 5 == 1: client_options.append("memory_tracker_fault_probability=0.001") + if i % 5 == 1: + client_options.append("merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.05") + if i % 2 == 1 and not upgrade_check: client_options.append("group_by_use_nulls=1") diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 13d1200758e..6eac6e537ec 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -843,6 +843,7 @@ class SettingsRandomizer: "prefer_warmed_unmerged_parts_seconds": lambda: random.randint(0, 10), "use_page_cache_for_disks_without_file_cache": lambda: random.random() < 0.7, "page_cache_inject_eviction": lambda: random.random() < 0.5, + "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability": lambda : round(random.random(), 2) } @staticmethod diff --git a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql index 61263d125b0..2266da5fc8f 100644 --- a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql +++ b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql @@ -1,6 +1,4 @@ --- Tags: no-debug, no-parallel, long, no-s3-storage, no-random-merge-tree-settings - -SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; +-- Tags: no-debug, no-parallel, long, no-s3-storage, no-random-settings, no-random-merge-tree-settings DROP TABLE IF EXISTS table_with_single_pk; From 4dc985aa2fc9dcb0ded21e70e175ca41cd25e785 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 6 Mar 2024 12:20:51 +0300 Subject: [PATCH 0331/1165] Fixed style check --- tests/ci/stress.py | 4 +++- tests/clickhouse-test | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/ci/stress.py b/tests/ci/stress.py index dbc04835fae..e0601b86f00 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -53,7 +53,9 @@ def get_options(i: int, upgrade_check: bool) -> str: client_options.append("memory_tracker_fault_probability=0.001") if i % 5 == 1: - client_options.append("merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.05") + client_options.append( + "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.05" + ) if i % 2 == 1 and not upgrade_check: client_options.append("group_by_use_nulls=1") diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 6eac6e537ec..0dd408a2c8c 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -843,7 +843,9 @@ class SettingsRandomizer: "prefer_warmed_unmerged_parts_seconds": lambda: random.randint(0, 10), "use_page_cache_for_disks_without_file_cache": lambda: random.random() < 0.7, "page_cache_inject_eviction": lambda: random.random() < 0.5, - "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability": lambda : round(random.random(), 2) + "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability": lambda: round( + random.random(), 2 + ), } @staticmethod From 218655228887bed057c057fd89afbffef0b9681c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 6 Mar 2024 18:41:56 +0300 Subject: [PATCH 0332/1165] Updated SettingsChangesHistory --- src/Core/SettingsChangesHistory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index e8e38ca82f9..be31dd52b10 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -102,6 +102,7 @@ static std::map sett {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"}, {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"}, {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"}, + {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, @@ -131,7 +132,6 @@ static std::map sett {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, }}, {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, From d8a31160b4547e9e56eeb4af768272faacc35bf2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 11 Mar 2024 11:20:54 +0300 Subject: [PATCH 0333/1165] Fixed tests --- tests/queries/0_stateless/02302_s3_file_pruning.sql | 2 ++ tests/queries/0_stateless/02987_group_array_intersect.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/02302_s3_file_pruning.sql b/tests/queries/0_stateless/02302_s3_file_pruning.sql index 93fc8a1bc25..647dfd5e5eb 100644 --- a/tests/queries/0_stateless/02302_s3_file_pruning.sql +++ b/tests/queries/0_stateless/02302_s3_file_pruning.sql @@ -1,6 +1,8 @@ -- Tags: no-parallel, no-fasttest -- Tag no-fasttest: Depends on S3 +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + -- { echo } drop table if exists test_02302; create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302_{_partition_id}', format=Parquet) partition by a; diff --git a/tests/queries/0_stateless/02987_group_array_intersect.sql b/tests/queries/0_stateless/02987_group_array_intersect.sql index 703914e464d..321e860b0a8 100644 --- a/tests/queries/0_stateless/02987_group_array_intersect.sql +++ b/tests/queries/0_stateless/02987_group_array_intersect.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test_empty; CREATE TABLE test_empty (a Array(Int64)) engine=MergeTree ORDER BY a; INSERT INTO test_empty VALUES ([]); From a5d286dc27ca3d9d32217be3c4d7fdd55ec82b0c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 13 Mar 2024 18:13:34 +0300 Subject: [PATCH 0334/1165] Fixed tests --- tests/queries/0_stateless/01508_partition_pruning_long.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.sh b/tests/queries/0_stateless/01508_partition_pruning_long.sh index c1f2d6562ab..b951e550708 100755 --- a/tests/queries/0_stateless/01508_partition_pruning_long.sh +++ b/tests/queries/0_stateless/01508_partition_pruning_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-polymorphic-parts, no-random-merge-tree-settings +# Tags: long, no-polymorphic-parts, no-random-settings, no-random-merge-tree-settings # Description of test result: # Test the correctness of the partition pruning From f162ea83417ab93a6883ad5f3ac9daa80e013dcd Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 21 Mar 2024 09:51:10 +0000 Subject: [PATCH 0335/1165] Fix use-of-uninitialized-value in parseDateTimeBestEffort --- src/IO/parseDateTimeBestEffort.cpp | 3 +++ tests/queries/0_stateless/03014_msan_parse_date_time.reference | 0 tests/queries/0_stateless/03014_msan_parse_date_time.sql | 1 + 3 files changed, 4 insertions(+) create mode 100644 tests/queries/0_stateless/03014_msan_parse_date_time.reference create mode 100644 tests/queries/0_stateless/03014_msan_parse_date_time.sql diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index caf51d94bb3..83928b32f2f 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -147,6 +147,9 @@ ReturnType parseDateTimeBestEffortImpl( { has_comma_between_date_and_time = true; ++in.position(); + + if (in.eof()) + break; } } diff --git a/tests/queries/0_stateless/03014_msan_parse_date_time.reference b/tests/queries/0_stateless/03014_msan_parse_date_time.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03014_msan_parse_date_time.sql b/tests/queries/0_stateless/03014_msan_parse_date_time.sql new file mode 100644 index 00000000000..d6daea69cfa --- /dev/null +++ b/tests/queries/0_stateless/03014_msan_parse_date_time.sql @@ -0,0 +1 @@ +SELECT parseDateTimeBestEffort(toFixedString('01/12/2017,', 11)); -- { serverError CANNOT_PARSE_DATETIME } From e0d14a1eaf2028ca806ebef0a64b555798a57988 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 21 Mar 2024 11:10:54 +0100 Subject: [PATCH 0336/1165] Updated lambda and name of BufferAllocationPolicy --- src/Common/BufferAllocationPolicy.cpp | 12 +++++----- src/Common/BufferAllocationPolicy.h | 12 +++++----- .../IO/WriteBufferFromAzureBlobStorage.cpp | 22 +++++++++---------- .../IO/WriteBufferFromAzureBlobStorage.h | 2 +- src/IO/WriteBufferFromS3.cpp | 6 ++--- src/IO/WriteBufferFromS3.h | 3 +-- 6 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/Common/BufferAllocationPolicy.cpp b/src/Common/BufferAllocationPolicy.cpp index 359da0f8313..980cbcca729 100644 --- a/src/Common/BufferAllocationPolicy.cpp +++ b/src/Common/BufferAllocationPolicy.cpp @@ -5,13 +5,13 @@ namespace DB { -class FixedSizeBufferAllocationPolicy : public IBufferAllocationPolicy +class FixedSizeBufferAllocationPolicy : public BufferAllocationPolicy { const size_t buffer_size = 0; size_t buffer_number = 0; public: - explicit FixedSizeBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_) + explicit FixedSizeBufferAllocationPolicy(const BufferAllocationPolicy::Settings & settings_) : buffer_size(settings_.strict_size) { chassert(buffer_size > 0); @@ -32,7 +32,7 @@ public: }; -class ExpBufferAllocationPolicy : public DB::IBufferAllocationPolicy +class ExpBufferAllocationPolicy : public DB::BufferAllocationPolicy { const size_t first_size = 0; const size_t second_size = 0; @@ -45,7 +45,7 @@ class ExpBufferAllocationPolicy : public DB::IBufferAllocationPolicy size_t buffer_number = 0; public: - explicit ExpBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_) + explicit ExpBufferAllocationPolicy(const BufferAllocationPolicy::Settings & settings_) : first_size(std::max(settings_.max_single_size, settings_.min_size)) , second_size(settings_.min_size) , multiply_factor(settings_.multiply_factor) @@ -89,9 +89,9 @@ public: }; -IBufferAllocationPolicy::~IBufferAllocationPolicy() = default; +BufferAllocationPolicy::~BufferAllocationPolicy() = default; -IBufferAllocationPolicy::IBufferAllocationPolicyPtr IBufferAllocationPolicy::create(IBufferAllocationPolicy::Settings settings_) +BufferAllocationPolicyPtr BufferAllocationPolicy::create(BufferAllocationPolicy::Settings settings_) { if (settings_.strict_size > 0) return std::make_unique(settings_); diff --git a/src/Common/BufferAllocationPolicy.h b/src/Common/BufferAllocationPolicy.h index 4ac20f1605b..7017891f9e2 100644 --- a/src/Common/BufferAllocationPolicy.h +++ b/src/Common/BufferAllocationPolicy.h @@ -9,8 +9,11 @@ namespace DB { +class BufferAllocationPolicy; +using BufferAllocationPolicyPtr = std::unique_ptr; + /// Buffer number starts with 0 -class IBufferAllocationPolicy +class BufferAllocationPolicy { public: @@ -24,15 +27,12 @@ public: size_t max_single_size = 32 * 1024 * 1024; /// Max size for a single buffer/block }; - virtual size_t getBufferNumber() const = 0; virtual size_t getBufferSize() const = 0; virtual void nextBuffer() = 0; - virtual ~IBufferAllocationPolicy() = 0; + virtual ~BufferAllocationPolicy() = 0; - using IBufferAllocationPolicyPtr = std::unique_ptr; - - static IBufferAllocationPolicyPtr create(Settings settings_); + static BufferAllocationPolicyPtr create(Settings settings_); }; diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index 1ef54272295..ed3ce76bd94 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -22,11 +22,12 @@ struct WriteBufferFromAzureBlobStorage::PartData { Memory<> memory; size_t data_size = 0; + std::string block_id; }; -IBufferAllocationPolicy::IBufferAllocationPolicyPtr createBufferAllocationPolicy(const AzureObjectStorageSettings & settings) +BufferAllocationPolicyPtr createBufferAllocationPolicy(const AzureObjectStorageSettings & settings) { - IBufferAllocationPolicy::Settings allocation_settings; + BufferAllocationPolicy::Settings allocation_settings; allocation_settings.strict_size = settings.strict_upload_part_size; allocation_settings.min_size = settings.min_upload_part_size; allocation_settings.max_size = settings.max_upload_part_size; @@ -34,7 +35,7 @@ IBufferAllocationPolicy::IBufferAllocationPolicyPtr createBufferAllocationPolicy allocation_settings.multiply_parts_count_threshold = settings.upload_part_size_multiply_parts_count_threshold; allocation_settings.max_single_size = settings.max_single_part_upload_size; - return IBufferAllocationPolicy::create(allocation_settings); + return BufferAllocationPolicy::create(allocation_settings); } WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( @@ -146,21 +147,20 @@ void WriteBufferFromAzureBlobStorage::allocateBuffer() void WriteBufferFromAzureBlobStorage::writePart() { - std::shared_ptr part_data; auto data_size = size_t(position() - memory.data()); - part_data = std::make_shared(std::move(memory), data_size); - WriteBuffer::set(nullptr, 0); - - if (part_data->data_size == 0) + if (data_size == 0) return; - auto upload_worker = [&, part_data] () + const std::string & block_id = block_ids.emplace_back(getRandomASCIIString(64)); + std::shared_ptr part_data = std::make_shared(std::move(memory), data_size, block_id); + WriteBuffer::set(nullptr, 0); + + auto upload_worker = [this, part_data] () { auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); - const std::string & block_id = block_ids.emplace_back(getRandomASCIIString(64)); Azure::Core::IO::MemoryBodyStream memory_stream(reinterpret_cast(part_data->memory.data()), part_data->data_size); - execWithRetry([&](){ block_blob_client.StageBlock(block_id, memory_stream); }, max_unexpected_write_error_retries, part_data->data_size); + execWithRetry([&](){ block_blob_client.StageBlock(part_data->block_id, memory_stream); }, max_unexpected_write_error_retries, part_data->data_size); if (write_settings.remote_throttler) write_settings.remote_throttler->add(part_data->data_size, ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds); diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index a210a75030a..6e10c07b255 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -58,7 +58,7 @@ private: LoggerPtr log; LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); - IBufferAllocationPolicy::IBufferAllocationPolicyPtr buffer_allocation_policy; + BufferAllocationPolicyPtr buffer_allocation_policy; const size_t max_single_part_upload_size; const size_t max_unexpected_write_error_retries; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 3ee59c42079..865bac86ff5 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -74,9 +74,9 @@ struct WriteBufferFromS3::PartData } }; -IBufferAllocationPolicy::IBufferAllocationPolicyPtr createBufferAllocationPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings) +BufferAllocationPolicyPtr createBufferAllocationPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings) { - IBufferAllocationPolicy::Settings allocation_settings; + BufferAllocationPolicy::Settings allocation_settings; allocation_settings.strict_size = settings.strict_upload_part_size; allocation_settings.min_size = settings.min_upload_part_size; allocation_settings.max_size = settings.max_upload_part_size; @@ -84,7 +84,7 @@ IBufferAllocationPolicy::IBufferAllocationPolicyPtr createBufferAllocationPolicy allocation_settings.multiply_parts_count_threshold = settings.upload_part_size_multiply_parts_count_threshold; allocation_settings.max_single_size = settings.max_single_part_upload_size; - return IBufferAllocationPolicy::create(allocation_settings); + return BufferAllocationPolicy::create(allocation_settings); } diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 3d2aed74e88..0eab20891bf 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -85,7 +85,7 @@ private: LoggerPtr log = getLogger("WriteBufferFromS3"); LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); - IBufferAllocationPolicy::IBufferAllocationPolicyPtr buffer_allocation_policy; + BufferAllocationPolicyPtr buffer_allocation_policy; /// Upload in S3 is made in parts. /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts. @@ -109,7 +109,6 @@ private: size_t total_size = 0; size_t hidden_size = 0; -// class TaskTracker; std::unique_ptr task_tracker; BlobStorageLogWriterPtr blob_log; From 33169e72df181915ad058af7fb7d60c53805d84b Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 21 Mar 2024 10:34:10 +0000 Subject: [PATCH 0337/1165] Fix AMBIGUOUS_IDENTIFIER with join_use_nulls https://github.com/ClickHouse/ClickHouse/issues/59067#issuecomment-1912500556 --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 17 ++++++- ..._join_use_nulls_column_not_found.reference | 12 +++++ ...alyzer_join_use_nulls_column_not_found.sql | 50 +++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 6a5efa6532f..c8f3f8b884b 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -3302,6 +3302,21 @@ QueryTreeNodePtr checkIsMissedObjectJSONSubcolumn(const QueryTreeNodePtr & left_ return {}; } +/// Compare resolved identifiers considering columns that become nullable after JOIN +static bool resolvedIdenfiersFromJoinAreEquals( + const QueryTreeNodePtr & left_resolved_identifier, + const QueryTreeNodePtr & right_resolved_identifier, + const IdentifierResolveScope & scope) +{ + auto lit = scope.nullable_join_columns.find(left_resolved_identifier); + const auto & left_resolved_to_compare = lit != scope.nullable_join_columns.end() ? lit->second : left_resolved_identifier; + + auto rit = scope.nullable_join_columns.find(right_resolved_identifier); + const auto & right_resolved_to_compare = rit != scope.nullable_join_columns.end() ? rit->second : right_resolved_identifier; + + return left_resolved_to_compare->isEqual(*right_resolved_to_compare, IQueryTreeNode::CompareOptions{.compare_aliases = false}); +} + QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) @@ -3438,7 +3453,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo resolved_identifier = std::move(result_column_node); } - else if (left_resolved_identifier->isEqual(*right_resolved_identifier, IQueryTreeNode::CompareOptions{.compare_aliases = false})) + else if (resolvedIdenfiersFromJoinAreEquals(left_resolved_identifier, right_resolved_identifier, scope)) { const auto & identifier_path_part = identifier_lookup.identifier.front(); auto * left_resolved_identifier_column = left_resolved_identifier->as(); diff --git a/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.reference b/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.reference index a5c8806279f..d53d7235814 100644 --- a/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.reference +++ b/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.reference @@ -1,2 +1,14 @@ +-- 3 +-- 3 +-- +0 +-- +\N \N +-- + a +a a +-- +a a +\N \N diff --git a/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.sql b/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.sql index 7101d5c89ae..5d1afa2a274 100644 --- a/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.sql +++ b/tests/queries/0_stateless/02973_analyzer_join_use_nulls_column_not_found.sql @@ -1,5 +1,7 @@ SET join_use_nulls = 1; +SELECT '--'; + select c FROM ( select d2.c @@ -9,6 +11,8 @@ select c FROM ( ) ; +SELECT '--'; + with d1 as ( select 1 as a, @@ -29,3 +33,49 @@ joined as ( ) select c from joined; + +SELECT '--'; + +WITH + a AS ( SELECT 0 AS key, 'a' AS acol ), + b AS ( SELECT 2 AS key ) +SELECT a.key +FROM b +LEFT JOIN a ON 1 +LEFT JOIN a AS a1 ON 1 +; + +SELECT '--'; + +WITH + a AS ( SELECT 0 AS key, 'a' AS acol ), + b AS ( SELECT 2 AS key ) +SELECT a.acol, a1.acol +FROM b +LEFT JOIN a ON a.key = b.key +LEFT JOIN a AS a1 ON a1.key = a.key +; +SELECT '--'; + +WITH + a AS ( SELECT 0 AS key, 'a' AS acol ), + b AS ( SELECT 2 AS key ) +SELECT a.acol, a1.acol +FROM b +FULL JOIN a ON a.key = b.key +FULL JOIN a AS a1 ON a1.key = a.key +ORDER BY 1 +SETTINGS join_use_nulls = 0 +; + +SELECT '--'; + +WITH + a AS ( SELECT 0 AS key, 'a' AS acol ), + b AS ( SELECT 2 AS key ) +SELECT a.acol, a1.acol +FROM b +FULL JOIN a ON a.key = b.key +FULL JOIN a AS a1 ON a1.key = a.key +ORDER BY 1 +; From e002e42805a7832f0a54766bc483e6d1fdcf877e Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 21 Mar 2024 10:38:50 +0000 Subject: [PATCH 0338/1165] remove old nodes from node_to_projection_name when node become nullable --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index c8f3f8b884b..0ca9108bea0 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -3560,13 +3560,17 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo if (scope.join_use_nulls) { - auto it = node_to_projection_name.find(resolved_identifier); + auto projection_name_it = node_to_projection_name.find(resolved_identifier); auto nullable_resolved_identifier = convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side, scope); if (nullable_resolved_identifier) { resolved_identifier = nullable_resolved_identifier; - if (it != node_to_projection_name.end()) - node_to_projection_name.emplace(resolved_identifier, it->second); + /// Set the same projection name for new nullable node + if (projection_name_it != node_to_projection_name.end()) + { + node_to_projection_name.erase(projection_name_it); + node_to_projection_name.emplace(resolved_identifier, projection_name_it->second); + } } } @@ -4686,7 +4690,10 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I node = nullable_node; /// Set the same projection name for new nullable node if (projection_name_it != node_to_projection_name.end()) + { + node_to_projection_name.erase(projection_name_it); node_to_projection_name.emplace(node, projection_name_it->second); + } } } } From a91eaa78ee0ecd1e1fc78fbb8a0379403b599d9f Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Thu, 21 Mar 2024 11:46:44 +0100 Subject: [PATCH 0339/1165] Update src/Interpreters/ServerAsynchronousMetrics.cpp Co-authored-by: Antonio Andelic --- src/Interpreters/ServerAsynchronousMetrics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index d26cfcf9fe9..7703a352130 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -279,7 +279,7 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr bool is_system = db.first == DatabaseCatalog::SYSTEM_DATABASE; // Note that we skip not yet loaded tables, so metrics could possibly be lower than expected on fully loaded database just after server start if `async_load_databases = true`. - for (auto iterator = db.second->getTablesIterator(getContext(), {}, true); iterator->isValid(); iterator->next()) + for (auto iterator = db.second->getTablesIterator(getContext(), {}, /*skip_not_loaded=*/true); iterator->isValid(); iterator->next()) { ++total_number_of_tables; if (is_system) From 6d84ccd9f163427a9d169b6d972f8f5e62db5237 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 21 Mar 2024 11:12:36 +0000 Subject: [PATCH 0340/1165] fix tests/ci/style_check.py for removed files --- tests/ci/style_check.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 797c0ef12d0..f8e49202c8e 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -100,6 +100,8 @@ def is_python(file: Union[Path, str]) -> bool: """returns if the changed file in the repository is python script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 + if not os.path.exists(os.path.join(REPO_COPY, file)): + return False return bool( magic.from_file(os.path.join(REPO_COPY, file), mime=True) == "text/x-script.python" @@ -110,6 +112,8 @@ def is_shell(file: Union[Path, str]) -> bool: """returns if the changed file in the repository is shell script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 + if not os.path.exists(os.path.join(REPO_COPY, file)): + return False return bool( magic.from_file(os.path.join(REPO_COPY, file), mime=True) == "text/x-shellscript" From 6e260d9419eaea891d972cf37447872980479531 Mon Sep 17 00:00:00 2001 From: Oxide Computer Company Date: Wed, 22 Nov 2023 11:47:36 +0000 Subject: [PATCH 0341/1165] poco foundation: add illumos support --- base/poco/Foundation/src/Environment_UNIX.cpp | 6 +++--- base/poco/Foundation/src/NamedEvent_UNIX.cpp | 2 +- base/poco/Foundation/src/NamedMutex_UNIX.cpp | 2 +- base/poco/Net/CMakeLists.txt | 4 ++++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/base/poco/Foundation/src/Environment_UNIX.cpp b/base/poco/Foundation/src/Environment_UNIX.cpp index 202e5d88f83..faabb374778 100644 --- a/base/poco/Foundation/src/Environment_UNIX.cpp +++ b/base/poco/Foundation/src/Environment_UNIX.cpp @@ -281,15 +281,15 @@ void EnvironmentImpl::nodeIdImpl(NodeId& id) /// #include #if defined(sun) || defined(__sun) #include +#include +#include +#include #endif /// #include /// #include /// #include /// #include /// #include -/// #include -/// #include -/// #include /// #include diff --git a/base/poco/Foundation/src/NamedEvent_UNIX.cpp b/base/poco/Foundation/src/NamedEvent_UNIX.cpp index 978e6e0bc02..3cda4104c73 100644 --- a/base/poco/Foundation/src/NamedEvent_UNIX.cpp +++ b/base/poco/Foundation/src/NamedEvent_UNIX.cpp @@ -31,7 +31,7 @@ namespace Poco { -#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) +#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) || (POCO_OS == POCO_OS_SOLARIS) union semun { int val; diff --git a/base/poco/Foundation/src/NamedMutex_UNIX.cpp b/base/poco/Foundation/src/NamedMutex_UNIX.cpp index 6cfa1369c9d..d53d54d7bb5 100644 --- a/base/poco/Foundation/src/NamedMutex_UNIX.cpp +++ b/base/poco/Foundation/src/NamedMutex_UNIX.cpp @@ -31,7 +31,7 @@ namespace Poco { -#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) +#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) || (POCO_OS == POCO_OS_SOLARIS) union semun { int val; diff --git a/base/poco/Net/CMakeLists.txt b/base/poco/Net/CMakeLists.txt index 792045c9b43..50ffbdf905a 100644 --- a/base/poco/Net/CMakeLists.txt +++ b/base/poco/Net/CMakeLists.txt @@ -9,6 +9,10 @@ elseif (OS_DARWIN OR OS_FREEBSD) target_compile_definitions (_poco_net PUBLIC POCO_HAVE_FD_POLL) endif () +if (OS_SUNOS) + target_link_libraries (_poco_net PUBLIC socket nsl) +endif () + # TODO: remove these warning exclusions target_compile_options (_poco_net PRIVATE From 7abdc66e0d8f3be7f488cdcf9362f4b599f2a8ce Mon Sep 17 00:00:00 2001 From: Oxide Computer Company Date: Wed, 22 Nov 2023 11:47:33 +0000 Subject: [PATCH 0342/1165] contrib/c-ares: add illumos as a platform --- contrib/c-ares-cmake/CMakeLists.txt | 2 + contrib/c-ares-cmake/solaris/ares_build.h | 104 +++++ contrib/c-ares-cmake/solaris/ares_config.h | 503 +++++++++++++++++++++ 3 files changed, 609 insertions(+) create mode 100644 contrib/c-ares-cmake/solaris/ares_build.h create mode 100644 contrib/c-ares-cmake/solaris/ares_config.h diff --git a/contrib/c-ares-cmake/CMakeLists.txt b/contrib/c-ares-cmake/CMakeLists.txt index 86ab6f90260..daec96ff1b1 100644 --- a/contrib/c-ares-cmake/CMakeLists.txt +++ b/contrib/c-ares-cmake/CMakeLists.txt @@ -86,6 +86,8 @@ elseif (OS_DARWIN) target_compile_definitions(_c-ares PRIVATE -D_DARWIN_C_SOURCE) elseif (OS_FREEBSD) target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/freebsd") +elseif (OS_SUNOS) + target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/solaris") endif() add_library(ch_contrib::c-ares ALIAS _c-ares) diff --git a/contrib/c-ares-cmake/solaris/ares_build.h b/contrib/c-ares-cmake/solaris/ares_build.h new file mode 100644 index 00000000000..f42b59d07bd --- /dev/null +++ b/contrib/c-ares-cmake/solaris/ares_build.h @@ -0,0 +1,104 @@ +/* include/ares_build.h. Generated from ares_build.h.in by configure. */ +#ifndef __CARES_BUILD_H +#define __CARES_BUILD_H + + +/* Copyright (C) 2009 - 2021 by Daniel Stenberg et al + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose and without fee is hereby granted, provided + * that the above copyright notice appear in all copies and that both that + * copyright notice and this permission notice appear in supporting + * documentation, and that the name of M.I.T. not be used in advertising or + * publicity pertaining to distribution of the software without specific, + * written prior permission. M.I.T. makes no representations about the + * suitability of this software for any purpose. It is provided "as is" + * without express or implied warranty. + */ + +/* ================================================================ */ +/* NOTES FOR CONFIGURE CAPABLE SYSTEMS */ +/* ================================================================ */ + +/* + * NOTE 1: + * ------- + * + * Nothing in this file is intended to be modified or adjusted by the + * c-ares library user nor by the c-ares library builder. + * + * If you think that something actually needs to be changed, adjusted + * or fixed in this file, then, report it on the c-ares development + * mailing list: http://lists.haxx.se/listinfo/c-ares/ + * + * This header file shall only export symbols which are 'cares' or 'CARES' + * prefixed, otherwise public name space would be polluted. + * + * NOTE 2: + * ------- + * + * Right now you might be staring at file ares_build.h.in or ares_build.h, + * this is due to the following reason: + * + * On systems capable of running the configure script, the configure process + * will overwrite the distributed ares_build.h file with one that is suitable + * and specific to the library being configured and built, which is generated + * from the ares_build.h.in template file. + * + */ + +/* ================================================================ */ +/* DEFINITION OF THESE SYMBOLS SHALL NOT TAKE PLACE ANYWHERE ELSE */ +/* ================================================================ */ + +#ifdef CARES_TYPEOF_ARES_SOCKLEN_T +# error "CARES_TYPEOF_ARES_SOCKLEN_T shall not be defined except in ares_build.h" + Error Compilation_aborted_CARES_TYPEOF_ARES_SOCKLEN_T_already_defined +#endif + +#define CARES_HAVE_ARPA_NAMESER_H 1 +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H 1 + +/* ================================================================ */ +/* EXTERNAL INTERFACE SETTINGS FOR CONFIGURE CAPABLE SYSTEMS ONLY */ +/* ================================================================ */ + +/* Configure process defines this to 1 when it finds out that system */ +/* header file ws2tcpip.h must be included by the external interface. */ +/* #undef CARES_PULL_WS2TCPIP_H */ +#ifdef CARES_PULL_WS2TCPIP_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# include +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/types.h must be included by the external interface. */ +#define CARES_PULL_SYS_TYPES_H 1 +#ifdef CARES_PULL_SYS_TYPES_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/socket.h must be included by the external interface. */ +#define CARES_PULL_SYS_SOCKET_H 1 +#ifdef CARES_PULL_SYS_SOCKET_H +# include +#endif + +/* Integral data type used for ares_socklen_t. */ +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t + +/* Data type definition of ares_socklen_t. */ +typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; + +/* Integral data type used for ares_ssize_t. */ +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Data type definition of ares_ssize_t. */ +typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; + +#endif /* __CARES_BUILD_H */ diff --git a/contrib/c-ares-cmake/solaris/ares_config.h b/contrib/c-ares-cmake/solaris/ares_config.h new file mode 100644 index 00000000000..c4ac5e38966 --- /dev/null +++ b/contrib/c-ares-cmake/solaris/ares_config.h @@ -0,0 +1,503 @@ +/* src/lib/ares_config.h. Generated from ares_config.h.in by configure. */ +/* src/lib/ares_config.h.in. Generated from configure.ac by autoheader. */ + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define this if ares is built for a big endian system */ +/* #undef ARES_BIG_ENDIAN */ + +/* Defined for build that exposes internal static functions for testing. */ +/* #undef CARES_EXPOSE_STATICS */ + +/* a suitable file/device to read random data from */ +#define CARES_RANDOM_FILE "/dev/urandom" + +/* Defined for build with symbol hiding. */ +#define CARES_SYMBOL_HIDING 1 + +/* Definition to make a library symbol externally visible. */ +#define CARES_SYMBOL_SCOPE_EXTERN __attribute__ ((__visibility__ ("default"))) + +/* the signed version of size_t */ +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Use resolver library to configure cares */ +/* #undef CARES_USE_LIBRESOLV */ + +/* if a /etc/inet dir is being used */ +#define ETC_INET 1 + +/* Define to the type of arg 2 for gethostname. */ +#define GETHOSTNAME_TYPE_ARG2 int + +/* Define to the type qualifier of arg 1 for getnameinfo. */ +#define GETNAMEINFO_QUAL_ARG1 const + +/* Define to the type of arg 1 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG1 struct sockaddr * + +/* Define to the type of arg 2 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG2 socklen_t + +/* Define to the type of args 4 and 6 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG46 socklen_t + +/* Define to the type of arg 7 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG7 int + +/* Specifies the number of arguments to getservbyport_r */ +#define GETSERVBYPORT_R_ARGS 5 + +/* Specifies the size of the buffer to pass to getservbyport_r */ +#define GETSERVBYPORT_R_BUFSIZE 4096 + +/* Define to 1 if you have AF_INET6. */ +#define HAVE_AF_INET6 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_COMPAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bitncmp' function. */ +/* #undef HAVE_BITNCMP */ + +/* Define to 1 if bool is an available type. */ +#define HAVE_BOOL_T 1 + +/* Define to 1 if you have the clock_gettime function and monotonic timer. */ +#define HAVE_CLOCK_GETTIME_MONOTONIC 1 + +/* Define to 1 if you have the closesocket function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the CloseSocket camel case function. */ +/* #undef HAVE_CLOSESOCKET_CAMEL */ + +/* Define to 1 if you have the connect function. */ +#define HAVE_CONNECT 1 + +/* define if the compiler supports basic C++11 syntax */ +#define HAVE_CXX11 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the fcntl function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* Define to 1 if you have a working fcntl O_NONBLOCK function. */ +#define HAVE_FCNTL_O_NONBLOCK 1 + +/* Define to 1 if you have the freeaddrinfo function. */ +#define HAVE_FREEADDRINFO 1 + +/* Define to 1 if you have a working getaddrinfo function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if the getaddrinfo function is threadsafe. */ +#define HAVE_GETADDRINFO_THREADSAFE 1 + +/* Define to 1 if you have the getenv function. */ +#define HAVE_GETENV 1 + +/* Define to 1 if you have the gethostbyaddr function. */ +#define HAVE_GETHOSTBYADDR 1 + +/* Define to 1 if you have the gethostbyname function. */ +#define HAVE_GETHOSTBYNAME 1 + +/* Define to 1 if you have the gethostname function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the getnameinfo function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the getservbyport_r function. */ +#define HAVE_GETSERVBYPORT_R 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the `if_indextoname' function. */ +#define HAVE_IF_INDEXTONAME 1 + +/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */ +/* #undef HAVE_INET_NET_PTON */ + +/* Define to 1 if you have a IPv6 capable working inet_ntop function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have a IPv6 capable working inet_pton function. */ +#define HAVE_INET_PTON 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the ioctl function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the ioctlsocket function. */ +/* #undef HAVE_IOCTLSOCKET */ + +/* Define to 1 if you have the IoctlSocket camel case function. */ +/* #undef HAVE_IOCTLSOCKET_CAMEL */ + +/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function. + */ +/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */ + +/* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* #undef HAVE_IOCTLSOCKET_FIONBIO */ + +/* Define to 1 if you have a working ioctl FIONBIO function. */ +/* #undef HAVE_IOCTL_FIONBIO */ + +/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */ +/* #undef HAVE_IOCTL_SIOCGIFADDR */ + +/* Define to 1 if you have the `resolve' library (-lresolve). */ +/* #undef HAVE_LIBRESOLVE */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if your compiler supports LL */ +#define HAVE_LL 1 + +/* Define to 1 if the compiler supports the 'long long' data type. */ +#define HAVE_LONGLONG 1 + +/* Define to 1 if you have the malloc.h header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the memory.h header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the MSG_NOSIGNAL flag. */ +#define HAVE_MSG_NOSIGNAL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H 1 + +/* Define to 1 if you have PF_INET6. */ +#define HAVE_PF_INET6 1 + +/* Define to 1 if you have the recv function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the recvfrom function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the send function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the setsockopt function. */ +#define HAVE_SETSOCKOPT 1 + +/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */ +/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H 1 + +/* Define to 1 if sig_atomic_t is an available typedef. */ +#define HAVE_SIG_ATOMIC_T 1 + +/* Define to 1 if sig_atomic_t is already defined as volatile. */ +/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */ + +/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */ +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID 1 + +/* Define to 1 if you have the socket function. */ +#define HAVE_SOCKET 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SOCKET_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDBOOL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the strcasecmp function. */ +#define HAVE_STRCASECMP 1 + +/* Define to 1 if you have the strcmpi function. */ +/* #undef HAVE_STRCMPI */ + +/* Define to 1 if you have the strdup function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the stricmp function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the strncasecmp function. */ +#define HAVE_STRNCASECMP 1 + +/* Define to 1 if you have the strncmpi function. */ +/* #undef HAVE_STRNCMPI */ + +/* Define to 1 if you have the strnicmp function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STROPTS_H 1 + +/* Define to 1 if you have struct addrinfo. */ +#define HAVE_STRUCT_ADDRINFO 1 + +/* Define to 1 if you have struct in6_addr. */ +#define HAVE_STRUCT_IN6_ADDR 1 + +/* Define to 1 if you have struct sockaddr_in6. */ +#define HAVE_STRUCT_SOCKADDR_IN6 1 + +/* if struct sockaddr_storage is defined */ +#define HAVE_STRUCT_SOCKADDR_STORAGE 1 + +/* Define to 1 if you have the timeval struct. */ +#define HAVE_STRUCT_TIMEVAL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the windows.h header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the winsock2.h header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the winsock.h header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the writev function. */ +#define HAVE_WRITEV 1 + +/* Define to 1 if you have the ws2tcpip.h header file. */ +/* #undef HAVE_WS2TCPIP_H */ + +/* Define if __system_property_get exists. */ +/* #undef HAVE___SYSTEM_PROPERTY_GET */ + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* Define to 1 if you need the malloc.h header file even with stdlib.h */ +/* #undef NEED_MALLOC_H */ + +/* Define to 1 if you need the memory.h header file even with stdlib.h */ +/* #undef NEED_MEMORY_H */ + +/* Define to 1 if _REENTRANT preprocessor symbol must be defined. */ +#define NEED_REENTRANT 1 + +/* Define to 1 if _THREAD_SAFE preprocessor symbol must be defined. */ +/* #undef NEED_THREAD_SAFE */ + +/* cpu-machine-OS */ +#define OS "x86_64-pc-solaris2.11" + +/* Name of package */ +#define PACKAGE "c-ares" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "c-ares mailing list: http://lists.haxx.se/listinfo/c-ares" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "c-ares" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "c-ares 1.18.1" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "c-ares" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "1.18.1" + +/* Define to the type qualifier pointed by arg 5 for recvfrom. */ +#define RECVFROM_QUAL_ARG5 + +/* Define to the type of arg 1 for recvfrom. */ +#define RECVFROM_TYPE_ARG1 int + +/* Define to the type pointed by arg 2 for recvfrom. */ +#define RECVFROM_TYPE_ARG2 void + +/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG2_IS_VOID 1 + +/* Define to the type of arg 3 for recvfrom. */ +#define RECVFROM_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recvfrom. */ +#define RECVFROM_TYPE_ARG4 int + +/* Define to the type pointed by arg 5 for recvfrom. */ +#define RECVFROM_TYPE_ARG5 struct sockaddr + +/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */ +/* #undef RECVFROM_TYPE_ARG5_IS_VOID */ + +/* Define to the type pointed by arg 6 for recvfrom. */ +#define RECVFROM_TYPE_ARG6 void + +/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG6_IS_VOID 1 + +/* Define to the function return type for recvfrom. */ +#define RECVFROM_TYPE_RETV ssize_t + +/* Define to the type of arg 1 for recv. */ +#define RECV_TYPE_ARG1 int + +/* Define to the type of arg 2 for recv. */ +#define RECV_TYPE_ARG2 void * + +/* Define to the type of arg 3 for recv. */ +#define RECV_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recv. */ +#define RECV_TYPE_ARG4 int + +/* Define to the function return type for recv. */ +#define RECV_TYPE_RETV ssize_t + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type qualifier of arg 2 for send. */ +#define SEND_QUAL_ARG2 const + +/* Define to the type of arg 1 for send. */ +#define SEND_TYPE_ARG1 int + +/* Define to the type of arg 2 for send. */ +#define SEND_TYPE_ARG2 void * + +/* Define to the type of arg 3 for send. */ +#define SEND_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for send. */ +#define SEND_TYPE_ARG4 int + +/* Define to the function return type for send. */ +#define SEND_TYPE_RETV ssize_t + +/* Define to 1 if all of the C90 standard headers exist (not just the ones + required in a freestanding environment). This macro is provided for + backward compatibility; new code need not use it. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . This + macro is obsolete. */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to disable non-blocking sockets. */ +/* #undef USE_BLOCKING_SOCKETS */ + +/* Version number of package */ +#define VERSION "1.18.1" + +/* Define to avoid automatic inclusion of winsock.h */ +/* #undef WIN32_LEAN_AND_MEAN */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to 1 if OS is AIX. */ +#ifndef _ALL_SOURCE +/* # undef _ALL_SOURCE */ +#endif + +/* Number of bits in a file offset, on hosts where this is settable. */ +/* #undef _FILE_OFFSET_BITS */ + +/* Define for large files, on AIX-style hosts. */ +/* #undef _LARGE_FILES */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Type to use in place of in_addr_t when system does not provide it. */ +/* #undef in_addr_t */ + +/* Define to `unsigned int' if does not define. */ +/* #undef size_t */ From 6a28e7ec9ce789b5d1684f6fb92365a1ac244aed Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 12:24:59 +0100 Subject: [PATCH 0343/1165] Fix #ci_set_analyzer --- .../02967_parallel_replicas_join_algo_and_analyzer.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference index 165504174d0..d7fa419aeab 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference @@ -82,8 +82,8 @@ nested join with analyzer and parallel replicas, both local 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both global From f64001ebfab4e7327d754ac139cbd8674b5d0ffe Mon Sep 17 00:00:00 2001 From: Oxide Computer Company Date: Thu, 21 Mar 2024 11:37:26 +0000 Subject: [PATCH 0344/1165] contrib/curl: Add illumos support --- contrib/curl-cmake/curl_config.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/curl-cmake/curl_config.h b/contrib/curl-cmake/curl_config.h index a38aa60fe6d..4d4c2972f57 100644 --- a/contrib/curl-cmake/curl_config.h +++ b/contrib/curl-cmake/curl_config.h @@ -51,3 +51,8 @@ #define USE_OPENSSL #define USE_THREADS_POSIX #define USE_ARES + +#ifdef __illumos__ +#define HAVE_POSIX_STRERROR_R 1 +#define HAVE_STRERROR_R 1 +#endif From 4700222a0d363a978b12ceb486930679ecd63442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 21 Mar 2024 12:40:07 +0100 Subject: [PATCH 0345/1165] Fuzzer: Try a different way to wait for the server --- docker/test/fuzzer/run-fuzzer.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 55f4d379005..76661a5b51c 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -181,7 +181,15 @@ function fuzz -- --path db \ --logger.console=0 \ --logger.log=server.log 2>&1 | tee -a stderr.log >> server.log 2>&1 & - server_pid=$(pidof clickhouse-server) + for _ in {1..30} + do + if clickhouse-client --query "select 1" + then + break + fi + sleep 1 + done + server_pid=$(cat /var/run/clickhouse-server/clickhouse-server.pid) kill -0 $server_pid From b1e03abc1f64fae40d0f829247396249a75a5760 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 21 Mar 2024 12:33:08 +0100 Subject: [PATCH 0346/1165] One more fix for SLRU --- src/Interpreters/Cache/IFileCachePriority.cpp | 5 +++ .../Cache/SLRUFileCachePriority.cpp | 35 +++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/Cache/IFileCachePriority.cpp b/src/Interpreters/Cache/IFileCachePriority.cpp index 9ef245dd8d2..6532af55ed2 100644 --- a/src/Interpreters/Cache/IFileCachePriority.cpp +++ b/src/Interpreters/Cache/IFileCachePriority.cpp @@ -10,6 +10,11 @@ namespace CurrentMetrics namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + IFileCachePriority::IFileCachePriority(size_t max_size_, size_t max_elements_) : max_size(max_size_), max_elements(max_elements_) { diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 9ea4cffeec0..07d04ee7876 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -10,6 +10,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { size_t getRatio(size_t total, double ratio) @@ -174,6 +179,8 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( return false; } + chassert(downgrade_candidates->size() > 0); + const size_t size_to_downgrade = stat.stat.releasable_size; const size_t elements_to_downgrade = stat.stat.releasable_count; @@ -182,39 +189,39 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( FileCacheReserveStat probationary_stat; bool downgrade_reached_size_limit = false; bool downgrade_reached_elements_limit = false; - if (!probationary_queue.canFit(size_to_downgrade, elements_to_downgrade, lock) - && !probationary_queue.collectCandidatesForEviction( + + if (!probationary_queue.collectCandidatesForEviction( size_to_downgrade, probationary_stat, res, reservee, user_id, downgrade_reached_size_limit, downgrade_reached_elements_limit, lock)) { return false; } - if (downgrade_candidates->size() == 0) - { - return true; - } + const size_t size_to_evict_from_probationary = probationary_stat.stat.releasable_size; + const size_t elements_to_evict_from_probationary = probationary_stat.stat.releasable_count; + + std::shared_ptr hold_space; const bool downgrade_after_eviction = res.size() > 0; - auto take_space_hold = [&]() + if (downgrade_after_eviction) { const size_t hold_size = downgrade_reached_size_limit - ? size_to_downgrade > probationary_stat.stat.releasable_size ? size - probationary_stat.stat.releasable_size : 0 + ? size_to_downgrade > size_to_evict_from_probationary ? size_to_downgrade - size_to_evict_from_probationary : 0 : size_to_downgrade; const size_t hold_elements = downgrade_reached_elements_limit - ? elements_to_downgrade > probationary_stat.stat.releasable_count ? size - probationary_stat.stat.releasable_count : 0 + ? elements_to_downgrade > elements_to_evict_from_probationary ? elements_to_downgrade - elements_to_evict_from_probationary : 0 : elements_to_downgrade; - return std::make_shared( + hold_space = std::make_shared( hold_size, hold_elements, QueueEntryType::SLRU_Probationary, probationary_queue, lock); - }; + } - auto downgrade_func = [=, holder = downgrade_after_eviction ? take_space_hold() : nullptr, this] + auto downgrade_func = [=, this] (const CachePriorityGuard::Lock & lk) { - if (holder) - holder->release(); + if (hold_space) + hold_space->release(); LOG_TEST(log, "Downgrading {} elements from protected to probationary. " "Total size: {}", From 81fd904b54327578c5f66e4452c77456178621a9 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 21 Mar 2024 00:05:21 +0100 Subject: [PATCH 0347/1165] I hate REPLACE_RANGE and actual_part_name --- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 1 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 65 ++++++++++++------- ..._move_partition_inactive_replica.reference | 8 +++ .../02916_move_partition_inactive_replica.sql | 46 +++++++++++++ 4 files changed, 98 insertions(+), 22 deletions(-) create mode 100644 tests/queries/0_stateless/02916_move_partition_inactive_replica.reference create mode 100644 tests/queries/0_stateless/02916_move_partition_inactive_replica.sql diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index 054c576cfc5..7693f34cc1e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -93,6 +93,7 @@ struct ReplicatedMergeTreeLogEntryData MergeTreeDataPartFormat new_part_format; String block_id; /// For parts of level zero, the block identifier for deduplication (node name in /blocks/). mutable String actual_new_part_name; /// GET_PART could actually fetch a part covering 'new_part_name'. + mutable std::unordered_set replace_range_actual_new_part_names; /// Same as above, but for REPLACE_RANGE UUID new_part_uuid = UUIDHelpers::Nil; Strings source_parts; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 42f564f40da..ee4ed87d456 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -342,6 +342,11 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( /// NOTE actual_new_part_name is very confusing and error-prone. This approach must be fixed. removeCoveredPartsFromMutations(entry->actual_new_part_name, /*remove_part = */ false, /*remove_covered_parts = */ true); } + for (const auto & actual_part : entry->replace_range_actual_new_part_names) + { + LOG_TEST(log, "Entry {} has actual new part name {}, removing it from mutations", entry->znode_name, actual_part); + removeCoveredPartsFromMutations(actual_part, /*remove_part = */ false, /*remove_covered_parts = */ true); + } LOG_TEST(log, "Adding parts [{}] to current parts", fmt::join(entry_virtual_parts, ", ")); @@ -1180,9 +1185,9 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry if (entry_for_same_part_it != future_parts.end()) { const LogEntry & another_entry = *entry_for_same_part_it->second; - constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} " + constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} (actual part {})" "because another log entry {} of type {} for the same part ({}) is being processed."; - LOG_INFO(LogToStr(out_reason, log), fmt_string, entry.znode_name, entry.type, entry.new_part_name, + LOG_INFO(LogToStr(out_reason, log), fmt_string, entry.znode_name, entry.type, entry.new_part_name, new_part_name, another_entry.znode_name, another_entry.type, another_entry.new_part_name); return true; @@ -1198,6 +1203,7 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry auto result_part = MergeTreePartInfo::fromPartName(new_part_name, format_version); /// It can slow down when the size of `future_parts` is large. But it can not be large, since background pool is limited. + /// (well, it can actually, thanks to REPLACE_RANGE, but it's a rare case) for (const auto & future_part_elem : future_parts) { auto future_part = MergeTreePartInfo::fromPartName(future_part_elem.first, format_version); @@ -1608,26 +1614,39 @@ void ReplicatedMergeTreeQueue::CurrentlyExecuting::setActualPartName( std::unique_lock & state_lock, std::vector & covered_entries_to_wait) { - if (!entry.actual_new_part_name.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Entry actual part isn't empty yet. This is a bug."); + if (actual_part_name.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Actual part name is empty"); - entry.actual_new_part_name = actual_part_name; + if (!entry.actual_new_part_name.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Entry {} actual part isn't empty yet: '{}'. This is a bug.", + entry.znode_name, entry.actual_new_part_name); + + auto actual_part_info = MergeTreePartInfo::fromPartName(actual_part_name, queue.format_version); + for (const auto & other_part_name : entry.replace_range_actual_new_part_names) + if (!MergeTreePartInfo::fromPartName(other_part_name, queue.format_version).isDisjoint(actual_part_info)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Entry {} already has actual part {} non-disjoint with {}. This is a bug.", + entry.actual_new_part_name, other_part_name, actual_part_name); /// Check if it is the same (and already added) part. - if (entry.actual_new_part_name == entry.new_part_name) + if (actual_part_name == entry.new_part_name) return; - if (!queue.future_parts.emplace(entry.actual_new_part_name, entry.shared_from_this()).second) + if (!queue.future_parts.emplace(actual_part_name, entry.shared_from_this()).second) throw Exception(ErrorCodes::LOGICAL_ERROR, "Attaching already existing future part {}. This is a bug. " "It happened on attempt to execute {}: {}", - entry.actual_new_part_name, entry.znode_name, entry.toString()); + actual_part_name, entry.znode_name, entry.toString()); + + if (entry.type == LogEntry::REPLACE_RANGE) + entry.replace_range_actual_new_part_names.insert(actual_part_name); + else + entry.actual_new_part_name = actual_part_name; for (LogEntryPtr & covered_entry : covered_entries_to_wait) { if (&entry == covered_entry.get()) continue; - LOG_TRACE(queue.log, "Waiting for {} producing {} to finish before executing {} producing not disjoint part {}", - covered_entry->znode_name, covered_entry->new_part_name, entry.znode_name, entry.new_part_name); + LOG_TRACE(queue.log, "Waiting for {} producing {} to finish before executing {} producing not disjoint part {} (actual part {})", + covered_entry->znode_name, covered_entry->new_part_name, entry.znode_name, entry.new_part_name, actual_part_name); covered_entry->execution_complete.wait(state_lock, [&covered_entry] { return !covered_entry->currently_executing; }); } } @@ -1646,25 +1665,27 @@ ReplicatedMergeTreeQueue::CurrentlyExecuting::~CurrentlyExecuting() entry->currently_executing = false; entry->execution_complete.notify_all(); - for (const String & new_part_name : entry->getVirtualPartNames(queue.format_version)) + auto erase_and_check = [this](const String & part_name) { - if (!queue.future_parts.erase(new_part_name)) + if (!queue.future_parts.erase(part_name)) { - LOG_ERROR(queue.log, "Untagging already untagged future part {}. This is a bug.", new_part_name); + LOG_ERROR(queue.log, "Untagging already untagged future part {}. This is a bug.", part_name); assert(false); } - } + }; + + for (const String & new_part_name : entry->getVirtualPartNames(queue.format_version)) + erase_and_check(new_part_name); if (!entry->actual_new_part_name.empty()) - { - if (entry->actual_new_part_name != entry->new_part_name && !queue.future_parts.erase(entry->actual_new_part_name)) - { - LOG_ERROR(queue.log, "Untagging already untagged future part {}. This is a bug.", entry->actual_new_part_name); - assert(false); - } + erase_and_check(entry->actual_new_part_name); - entry->actual_new_part_name.clear(); - } + entry->actual_new_part_name.clear(); + + for (const auto & actual_part : entry->replace_range_actual_new_part_names) + erase_and_check(actual_part); + + entry->replace_range_actual_new_part_names.clear(); } diff --git a/tests/queries/0_stateless/02916_move_partition_inactive_replica.reference b/tests/queries/0_stateless/02916_move_partition_inactive_replica.reference new file mode 100644 index 00000000000..b64cc6289d5 --- /dev/null +++ b/tests/queries/0_stateless/02916_move_partition_inactive_replica.reference @@ -0,0 +1,8 @@ +all_0_0_0 0 +all_0_1_1 1 +all_1_1_0 0 +all_2_2_0 0 +all_2_3_1 1 +all_3_3_0 0 +0 +40 1580 diff --git a/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql new file mode 100644 index 00000000000..d597c037612 --- /dev/null +++ b/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql @@ -0,0 +1,46 @@ +-- Tags: no-parallel + +create database if not exists shard_0; +create database if not exists shard_1; + +drop table if exists shard_0.from_1; +drop table if exists shard_1.from_1; +drop table if exists shard_0.to; +drop table if exists shard_1.to; + +create table shard_0.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; +create table shard_1.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; + +system stop merges shard_0.from_1; +insert into shard_0.from_1 select number + 20 from numbers(10); +insert into shard_0.from_1 select number + 30 from numbers(10); + +insert into shard_0.from_1 select number + 40 from numbers(10); +insert into shard_0.from_1 select number + 50 from numbers(10); + +system sync replica shard_1.from_1; + +create table shard_0.to (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/to_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, max_parts_to_merge_at_once=2; + +create table shard_1.to (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/to_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, max_parts_to_merge_at_once=2; + +detach table shard_1.to; + +alter table shard_0.from_1 on cluster test_cluster_two_shards_different_databases move partition tuple() to table shard_0.to format Null settings distributed_ddl_output_mode='never_throw', distributed_ddl_task_timeout = 1; + +drop table if exists shard_0.from_1; +drop table if exists shard_1.from_1; +OPTIMIZE TABLE shard_0.to; +OPTIMIZE TABLE shard_0.to; +select name, active from system.parts where database='shard_0' and table='to' order by name; + +system restart replica shard_0.to; + +select sleep(3); + +attach table shard_1.to; +system sync replica shard_1.to; +select count(), sum(x) from shard_1.to; + +drop table if exists shard_0.to; +drop table if exists shard_1.to; From cb71537c62cf4cb87121bd8d3db0d0602fac7286 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 21 Mar 2024 13:11:56 +0100 Subject: [PATCH 0348/1165] Ping CI From 7708e0e093a333baf39e9eb3058548cd52eed244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 21 Mar 2024 13:12:59 +0100 Subject: [PATCH 0349/1165] Disable some tests for SMT --- .../0_stateless/03002_part_log_rmt_fetch_merge_error.sql | 3 ++- .../0_stateless/03002_part_log_rmt_fetch_mutate_error.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql index a7381df5e11..548a8e5570a 100644 --- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql +++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql @@ -1,4 +1,5 @@ --- Tags: no-replicated-database, no-parallel +-- Tags: no-replicated-database, no-parallel, no-shared-merge-tree +-- SMT: The merge process is completely different from RMT drop table if exists rmt_master; drop table if exists rmt_slave; diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql index f19cb9c73c9..34ba034f798 100644 --- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql +++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql @@ -1,4 +1,5 @@ --- Tags: no-replicated-database, no-parallel +-- Tags: no-replicated-database, no-parallel, no-shared-merge-tree +-- SMT: The merge process is completely different from RMT drop table if exists rmt_master; drop table if exists rmt_slave; From 638754f988cb70511801ec904af85dd4b763d92a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 21 Mar 2024 13:34:32 +0100 Subject: [PATCH 0350/1165] Fix signal handler for sanitizer signals --- src/Daemon/BaseDaemon.cpp | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 289a41bb75e..7fc210a691a 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -275,7 +275,10 @@ public: } readPODBinary(stack_trace, in); - readVectorBinary(thread_frame_pointers, in); + + if (sig != SanitizerTrap) + readVectorBinary(thread_frame_pointers, in); + readBinary(thread_num, in); readPODBinary(thread_ptr, in); @@ -542,6 +545,16 @@ private: #if defined(SANITIZER) + +template +struct ValueHolder +{ + ValueHolder(T value_) : value(value_) + {} + + T value; +}; + extern "C" void __sanitizer_set_death_callback(void (*)()); /// Sanitizers may not expect some function calls from death callback. @@ -559,10 +572,13 @@ static DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback() const StackTrace stack_trace; - int sig = SignalListener::SanitizerTrap; - writeBinary(sig, out); + writeBinary(SignalListener::SanitizerTrap, out); writePODBinary(stack_trace, out); - writeBinary(UInt32(getThreadId()), out); + /// We create a dummy struct with a constructor so DISABLE_SANITIZER_INSTRUMENTATION is not applied to it + /// otherwise, Memory sanitizer can't know that values initiialized inside this function are actually initialized + /// because instrumentations are disabled leading to false positives later on + ValueHolder thread_id{static_cast(getThreadId())}; + writeBinary(thread_id.value, out); writePODBinary(current_thread, out); out.next(); From 77e947c44f343d3aab32340c4fda4545fb794a57 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 21 Mar 2024 14:09:01 +0100 Subject: [PATCH 0351/1165] Avoid `IsADirectoryError: Is a directory contrib/azure` --- tests/ci/style_check.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 797c0ef12d0..2604793c900 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -100,20 +100,28 @@ def is_python(file: Union[Path, str]) -> bool: """returns if the changed file in the repository is python script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-script.python" - ) + try: + return bool( + magic.from_file(os.path.join(REPO_COPY, file), mime=True) + == "text/x-script.python" + ) + except IsADirectoryError: + # Process submodules w/o errors + return False def is_shell(file: Union[Path, str]) -> bool: """returns if the changed file in the repository is shell script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-shellscript" - ) + try: + return bool( + magic.from_file(os.path.join(REPO_COPY, file), mime=True) + == "text/x-shellscript" + ) + except IsADirectoryError: + # Process submodules w/o errors + return False def main(): From e828acd0e5f98d0f6c61c260be9cb428ab0695d6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 Mar 2024 13:17:15 +0000 Subject: [PATCH 0352/1165] Review fixes. --- src/Storages/System/StorageSystemDetachedParts.cpp | 6 +----- src/Storages/System/StorageSystemPartsBase.cpp | 10 ++-------- src/Storages/VirtualColumnUtils.cpp | 6 +++--- src/Storages/VirtualColumnUtils.h | 3 +++ 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index 7207e981561..ebcd8d63a52 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -329,7 +329,6 @@ protected: void ReadFromSystemDetachedParts::applyFilters(ActionDAGNodes added_filter_nodes) { - filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); if (filter_actions_dag) { @@ -344,10 +343,7 @@ void ReadFromSystemDetachedParts::applyFilters(ActionDAGNodes added_filter_nodes filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); if (filter) - { - auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); - VirtualColumnUtils::filterBlockWithDAG(filter, empty_block, context); - } + VirtualColumnUtils::buildSetsForDAG(filter, context); } } diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index c81c2c18b2e..11cd6989f31 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -264,10 +264,7 @@ void ReadFromSystemPartsBase::applyFilters(ActionDAGNodes added_filter_nodes) filter_by_database = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); if (filter_by_database) - { - auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); - VirtualColumnUtils::filterBlockWithDAG(filter_by_database, empty_block, context); - } + VirtualColumnUtils::buildSetsForDAG(filter_by_database, context); block.insert(ColumnWithTypeAndName({}, std::make_shared(), "table")); block.insert(ColumnWithTypeAndName({}, std::make_shared(), "engine")); @@ -276,10 +273,7 @@ void ReadFromSystemPartsBase::applyFilters(ActionDAGNodes added_filter_nodes) filter_by_other_columns = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); if (filter_by_other_columns) - { - auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); - VirtualColumnUtils::filterBlockWithDAG(filter_by_database, empty_block, context); - } + VirtualColumnUtils::buildSetsForDAG(filter_by_other_columns, context); } } diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index c3ac27903c9..e3cbff5f01b 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -53,9 +53,9 @@ namespace DB namespace VirtualColumnUtils { -static void makeSets(const ExpressionActionsPtr & actions, const ContextPtr & context) +void buildSetsForDAG(const ActionsDAGPtr & dag, const ContextPtr & context) { - for (const auto & node : actions->getNodes()) + for (const auto & node : dag->getNodes()) { if (node.type == ActionsDAG::ActionType::COLUMN) { @@ -78,8 +78,8 @@ static void makeSets(const ExpressionActionsPtr & actions, const ContextPtr & co void filterBlockWithDAG(ActionsDAGPtr dag, Block & block, ContextPtr context) { + buildSetsForDAG(dag, context); auto actions = std::make_shared(dag); - makeSets(actions, context); Block block_with_filter = block; actions->execute(block_with_filter, /*dry_run=*/ false, /*allow_duplicates_in_input=*/ true); diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h index 83494872cac..62f2e4855b5 100644 --- a/src/Storages/VirtualColumnUtils.h +++ b/src/Storages/VirtualColumnUtils.h @@ -25,6 +25,9 @@ void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, /// Just filters block. Block should contain all the required columns. void filterBlockWithDAG(ActionsDAGPtr dag, Block & block, ContextPtr context); +/// Builds sets used by ActionsDAG inplace. +void buildSetsForDAG(const ActionsDAGPtr & dag, const ContextPtr & context); + /// Recursively checks if all functions used in DAG are deterministic in scope of query. bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node); From 7db3d21ab7887a486950cb413358e9650369096e Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:19:17 +0100 Subject: [PATCH 0353/1165] Update src/Processors/Transforms/FillingTransform.cpp Co-authored-by: Igor Nikonov <954088+devcrafter@users.noreply.github.com> --- src/Processors/Transforms/FillingTransform.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index 54812405610..879d61bb241 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -250,7 +250,7 @@ FillingTransform::FillingTransform( if (!unique_positions.insert(pos).second) throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "Multiple WITH FILL for identical expressions is not supported in ORDER BY"); if (ordinary_sort_positions.contains(pos)) - throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "ORDER BY containing the same expression with and without WITH FILL is not supported"); + throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "ORDER BY containing the same expression with and without WITH FILL modifier is not supported"); } if (use_with_fill_by_sorting_prefix) From 15a61a998a176314debe152b5d59be4ff3f5797a Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 21 Mar 2024 14:34:31 +0100 Subject: [PATCH 0354/1165] Fix logic for run_cpp_check, it had a bug --- tests/ci/style_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 2604793c900..d7f6fa998e9 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -143,8 +143,8 @@ def main(): run_python_check = True if CI and pr_info.number > 0: pr_info.fetch_changed_files() - run_cpp_check = not any( - is_python(file) or is_shell(file) for file in pr_info.changed_files + run_cpp_check = any( + not (is_python(file) or is_shell(file)) for file in pr_info.changed_files ) run_shell_check = any(is_shell(file) for file in pr_info.changed_files) run_python_check = any(is_python(file) for file in pr_info.changed_files) From 48cb228c9a88b1f77c9ec10ce04107edec3e190c Mon Sep 17 00:00:00 2001 From: Shuai li Date: Thu, 21 Mar 2024 21:47:29 +0800 Subject: [PATCH 0355/1165] Fix addDays cause an error when used datetime64 (#61561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix addDays cause an error when used datetime64 * add exception describe * Update tests/queries/0_stateless/03013_addDays_with_timezone.sql * remove file be executable * fix timezone * fix ci --------- Co-authored-by: János Benjamin Antal --- .../FunctionDateOrDateTimeAddInterval.h | 4 +-- ...21_datetime64_compatibility_long.reference | 32 +++++++++---------- .../03013_addDays_with_timezone.reference | 1 + .../03013_addDays_with_timezone.sql | 1 + 4 files changed, 20 insertions(+), 18 deletions(-) create mode 100644 tests/queries/0_stateless/03013_addDays_with_timezone.reference create mode 100644 tests/queries/0_stateless/03013_addDays_with_timezone.sql diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index b8c0d27c42e..f50b1415622 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -621,9 +621,9 @@ public: } else { - if (!WhichDataType(arguments[0].type).isDateTime()) + if (!WhichDataType(arguments[0].type).isDateTimeOrDateTime64()) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of first argument of function {}. " - "Must be a DateTime", arguments[0].type->getName(), getName()); + "Must be a DateTime/DateTime64", arguments[0].type->getName(), getName()); if (!WhichDataType(arguments[2].type).isString()) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of third argument of function {}. " diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference index a946a114bf4..74b7b207661 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference @@ -241,82 +241,82 @@ SELECT toYYYYMMDDhhmmss(N, \'Asia/Istanbul\') SELECT addYears(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2020-09-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2020-09-16 19:20:11.234" ------------------------------------------ SELECT addMonths(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-10-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-10-16 19:20:11.234" ------------------------------------------ SELECT addWeeks(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-23 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-23 19:20:11.234" ------------------------------------------ SELECT addDays(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-17 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-17 19:20:11.234" ------------------------------------------ SELECT addHours(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 20:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 20:20:11.234" ------------------------------------------ SELECT addMinutes(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 19:21:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 19:21:11.234" ------------------------------------------ SELECT addSeconds(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 19:20:12" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT addQuarters(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-12-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-12-16 19:20:11.234" ------------------------------------------ SELECT subtractYears(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2018-09-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2018-09-16 19:20:11.234" ------------------------------------------ SELECT subtractMonths(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-08-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-08-16 19:20:11.234" ------------------------------------------ SELECT subtractWeeks(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-09 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-09 19:20:11.234" ------------------------------------------ SELECT subtractDays(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-15 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-15 19:20:11.234" ------------------------------------------ SELECT subtractHours(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 18:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 18:20:11.234" ------------------------------------------ SELECT subtractMinutes(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 19:19:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 19:19:11.234" ------------------------------------------ SELECT subtractSeconds(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 19:20:10" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT subtractQuarters(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-06-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-06-16 19:20:11.234" ------------------------------------------ SELECT CAST(N as DateTime(\'Europe/Minsk\')) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" diff --git a/tests/queries/0_stateless/03013_addDays_with_timezone.reference b/tests/queries/0_stateless/03013_addDays_with_timezone.reference new file mode 100644 index 00000000000..6a8fa4f8a2c --- /dev/null +++ b/tests/queries/0_stateless/03013_addDays_with_timezone.reference @@ -0,0 +1 @@ +2024-01-11 00:00:00.000000 diff --git a/tests/queries/0_stateless/03013_addDays_with_timezone.sql b/tests/queries/0_stateless/03013_addDays_with_timezone.sql new file mode 100644 index 00000000000..eb822d53898 --- /dev/null +++ b/tests/queries/0_stateless/03013_addDays_with_timezone.sql @@ -0,0 +1 @@ +select addDays(toDateTime64('2024-01-01', 6, 'Asia/Shanghai'), 10, 'Asia/Shanghai'); From 188bc3cbaf0eb64ed6bd41e3ff71454ee53b70e6 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 15:00:27 +0100 Subject: [PATCH 0356/1165] Update some tests --- .../test_backward_compatibility/test_convert_ordinary.py | 2 +- .../test_backward_compatibility/test_cte_distributed.py | 4 ++-- .../integration/test_backward_compatibility/test_functions.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_backward_compatibility/test_convert_ordinary.py b/tests/integration/test_backward_compatibility/test_convert_ordinary.py index 8e7d773ad2c..ff296d01c18 100644 --- a/tests/integration/test_backward_compatibility/test_convert_ordinary.py +++ b/tests/integration/test_backward_compatibility/test_convert_ordinary.py @@ -9,7 +9,7 @@ node = cluster.add_instance( stay_alive=True, with_zookeeper=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_backward_compatibility/test_cte_distributed.py b/tests/integration/test_backward_compatibility/test_cte_distributed.py index e612bf2989a..592ec4ecbcc 100644 --- a/tests/integration/test_backward_compatibility/test_cte_distributed.py +++ b/tests/integration/test_backward_compatibility/test_cte_distributed.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", with_zookeeper=False, allow_analyzer=False) +node1 = cluster.add_instance("node1", with_zookeeper=False, use_old_analyzer=True) node2 = cluster.add_instance( "node2", with_zookeeper=False, @@ -11,7 +11,7 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index 0217c46a660..d95c58085f3 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -9,13 +9,13 @@ from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) -upstream = cluster.add_instance("upstream", allow_analyzer=False) +upstream = cluster.add_instance("upstream", use_old_analyzer=True) backward = cluster.add_instance( "backward", image="clickhouse/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) From 466cfe29b27906bc458ddb2108e994e17a2534b9 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 15:01:08 +0100 Subject: [PATCH 0357/1165] Fix tests with clickhouse-local #ci_set_analyzer --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 +- tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 0414ea93f94..70990cc87ff 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -7134,7 +7134,7 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node, ColumnDescription column = insert_columns.get(*insert_column_name_it); /// Change ephemeral columns to default columns. column.default_desc.kind = ColumnDefaultKind::Default; - structure_hint.add(insert_columns.get(*insert_column_name_it)); + structure_hint.add(std::move(column)); } } diff --git a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh index acf0608acc5..eb4a91bd850 100755 --- a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh +++ b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh @@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 1" 2>&1 | grep -q "Code: 47. DB::Exception:" && echo 'OK' || echo 'FAIL' ||: -$CLICKHOUSE_LOCAL --query="SELECT n -- { serverError 47 }" 2>&1 | grep -o 'Missing columns' +$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 0 -- { serverError 47 }" 2>&1 | grep -o 'Missing columns' From 46ab53ef797023fa90ff1d15a91b02d77222e400 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 21 Mar 2024 14:03:43 +0000 Subject: [PATCH 0358/1165] fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 9aeb9cd6f23..12adb762a9a 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -3569,7 +3569,6 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo /// Set the same projection name for new nullable node if (projection_name_it != node_to_projection_name.end()) { - node_to_projection_name.erase(projection_name_it); node_to_projection_name.emplace(resolved_identifier, projection_name_it->second); } } @@ -4692,7 +4691,6 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I /// Set the same projection name for new nullable node if (projection_name_it != node_to_projection_name.end()) { - node_to_projection_name.erase(projection_name_it); node_to_projection_name.emplace(node, projection_name_it->second); } } From 1e41752c76e3d8a361f36e4de7016c7833f26023 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 15:07:56 +0100 Subject: [PATCH 0359/1165] Fix the last test with clickhouse local --- tests/queries/0_stateless/01600_detach_permanently.reference | 1 + tests/queries/0_stateless/01600_detach_permanently.sh | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01600_detach_permanently.reference b/tests/queries/0_stateless/01600_detach_permanently.reference index 98ed3b6762d..4fda43df162 100644 --- a/tests/queries/0_stateless/01600_detach_permanently.reference +++ b/tests/queries/0_stateless/01600_detach_permanently.reference @@ -96,6 +96,7 @@ View can be reattached test for MV with inner table MV is working 1 +1 View can be reattached ################## DETACH DATABASE is not implemented (proper error) diff --git a/tests/queries/0_stateless/01600_detach_permanently.sh b/tests/queries/0_stateless/01600_detach_permanently.sh index 036706d2fe8..6721dbf3015 100755 --- a/tests/queries/0_stateless/01600_detach_permanently.sh +++ b/tests/queries/0_stateless/01600_detach_permanently.sh @@ -111,7 +111,9 @@ clickhouse_local "INSERT INTO db_ordinary.src SELECT * FROM numbers(10)" clickhouse_local "SELECT if(count() = 10, 'MV is working', 'MV failed') FROM db_ordinary.src_mv_with_inner" clickhouse_local "DETACH VIEW db_ordinary.src_mv_with_inner PERMANENTLY; INSERT INTO db_ordinary.src SELECT * FROM numbers(10)" --stacktrace -clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner" 2>&1 | grep -c "db_ordinary.src_mv_with_inner does not exist" +clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner SETTINGS allow_experimental_analyzer = 0" 2>&1 | grep -c "db_ordinary.src_mv_with_inner does not exist" +clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner SETTINGS allow_experimental_analyzer = 1" 2>&1 | grep -c "Unknown table expression identifier 'db_ordinary.src_mv_with_inner'" + ## Quite silly: ATTACH MATERIALIZED VIEW don't work with short syntax (w/o select), but i can attach it using ATTACH TABLE ... clickhouse_local "ATTACH TABLE db_ordinary.src_mv_with_inner" clickhouse_local "INSERT INTO db_ordinary.src SELECT * FROM numbers(10)" From 7c173e4dbc9606f2d0f4de6d61bbfe214997891c Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 16:03:50 +0100 Subject: [PATCH 0360/1165] Analyzer: fix group_by_use_nulls #ci_set_analyzer --- src/Planner/PlannerExpressionAnalysis.cpp | 8 ++++++-- .../03014_analyzer_group_by_use_nulls.reference | 2 ++ .../0_stateless/03014_analyzer_group_by_use_nulls.sql | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.reference create mode 100644 tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.sql diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index c664317b250..441e2ac4b9e 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include @@ -111,7 +113,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP continue; auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; - available_columns_after_aggregation.emplace_back(expression_dag_node->column, expression_type_after_aggregation, expression_dag_node->result_name); + auto column_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; + available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name); @@ -161,7 +164,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP continue; auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; - available_columns_after_aggregation.emplace_back(expression_dag_node->column, expression_type_after_aggregation, expression_dag_node->result_name); + auto column_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; + available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name); diff --git a/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.reference b/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.reference new file mode 100644 index 00000000000..b50fdcee209 --- /dev/null +++ b/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.reference @@ -0,0 +1,2 @@ +a b +a b diff --git a/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.sql b/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.sql new file mode 100644 index 00000000000..a1c302465de --- /dev/null +++ b/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.sql @@ -0,0 +1 @@ +SELECT 'a' AS key, 'b' as value GROUP BY key WITH CUBE SETTINGS group_by_use_nulls = 1; From 20a860cf1912f76bf60f18aa8ccacc4b812077cc Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 16:18:49 +0100 Subject: [PATCH 0361/1165] Update ci_config.py --- tests/ci/ci_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 7c213da27ec..bd64712a162 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -94,7 +94,7 @@ class JobNames(metaclass=WithIter): STATELESS_TEST_MSAN = "Stateless tests (msan)" STATELESS_TEST_UBSAN = "Stateless tests (ubsan)" STATELESS_TEST_ANALYZER_S3_REPLICATED_RELEASE = ( - "Stateless tests (release, analyzer, s3, DatabaseReplicated)" + "Stateless tests (release, old analyzer, s3, DatabaseReplicated)" ) # merged into STATELESS_TEST_ANALYZER_S3_REPLICATED_RELEASE: # STATELESS_TEST_ANALYZER_RELEASE = "Stateless tests (release, analyzer)" @@ -127,7 +127,7 @@ class JobNames(metaclass=WithIter): INTEGRATION_TEST = "Integration tests (release)" INTEGRATION_TEST_ASAN = "Integration tests (asan)" - INTEGRATION_TEST_ASAN_ANALYZER = "Integration tests (asan, analyzer)" + INTEGRATION_TEST_ASAN_ANALYZER = "Integration tests (asan, old analyzer)" INTEGRATION_TEST_TSAN = "Integration tests (tsan)" INTEGRATION_TEST_ARM = "Integration tests (aarch64)" INTEGRATION_TEST_FLAKY = "Integration tests flaky check (asan)" From f36e19917fcb5d8acc290bee77ff1ffd9ceca587 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 21 Mar 2024 15:27:39 +0000 Subject: [PATCH 0362/1165] Disable analyzer for EXPLAIN SYNTAX queries --- src/Interpreters/InterpreterFactory.cpp | 4 ++++ ...allel_replicas_cte_explain_syntax_crash.sql | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 0b87ec3cc0e..387d056ffe7 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -219,6 +219,10 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte } else if (query->as()) { + const auto kind = query->as()->getKind(); + if (kind == ASTExplainQuery::ParsedAST || kind == ASTExplainQuery::AnalyzedSyntax) + context->setSetting("allow_experimental_analyzer", false); + interpreter_name = "InterpreterExplainQuery"; } else if (query->as()) diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql index f46817d5e82..df4ec9d26a3 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql +++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql @@ -3,29 +3,26 @@ DROP TABLE IF EXISTS numbers_1e6__fuzz_33; CREATE TABLE numbers_1e6__fuzz_34 ( - `n` LowCardinality(Nullable(UInt8)) + n UInt64 ) ENGINE = MergeTree ORDER BY n -SETTINGS allow_nullable_key = 1 AS SELECT * -FROM numbers(1000000) -SETTINGS allow_suspicious_low_cardinality_types = 1; +FROM numbers(10); CREATE TABLE numbers_1e6__fuzz_33 ( - `n` LowCardinality(Nullable(UInt8)) + n UInt64 ) ENGINE = MergeTree ORDER BY n -SETTINGS allow_nullable_key = 1 AS SELECT * -FROM numbers(1000000) -SETTINGS allow_suspicious_low_cardinality_types = 1; +FROM numbers(10); SET allow_experimental_analyzer = 1; -SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=0; + EXPLAIN SYNTAX WITH cte1 AS @@ -41,3 +38,6 @@ WITH ) SELECT count() FROM cte2; + +DROP TABLE numbers_1e6__fuzz_34; +DROP TABLE numbers_1e6__fuzz_33; From 37b34d6ed159b76528e5bc8b1d74705eae0ba97d Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 16:28:45 +0100 Subject: [PATCH 0363/1165] Analyzer: Clear list of broken integration tests --- tests/analyzer_integration_broken_tests.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/analyzer_integration_broken_tests.txt b/tests/analyzer_integration_broken_tests.txt index 56fde20c363..e69de29bb2d 100644 --- a/tests/analyzer_integration_broken_tests.txt +++ b/tests/analyzer_integration_broken_tests.txt @@ -1 +0,0 @@ -test_concurrent_backups_s3/test.py::test_concurrent_backups From d1d72778021e9e13dc4a9a3bf4a34b53eb55ce7e Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 21 Mar 2024 15:29:11 +0000 Subject: [PATCH 0364/1165] fix clang-tidy and test --- src/Common/ColumnsHashingImpl.h | 4 ++-- .../03009_consecutive_keys_nullable.reference | 6 +++--- .../0_stateless/03009_consecutive_keys_nullable.sql | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Common/ColumnsHashingImpl.h b/src/Common/ColumnsHashingImpl.h index c8d62fa7e6b..6b3cc24d870 100644 --- a/src/Common/ColumnsHashingImpl.h +++ b/src/Common/ColumnsHashingImpl.h @@ -67,7 +67,7 @@ template struct LastElementCache; template struct LastElementCache : public LastElementCacheBase { - Value value; + Value value{}; bool is_null = false; template @@ -79,7 +79,7 @@ struct LastElementCache : public LastElementCacheBase template struct LastElementCache : public LastElementCacheBase { - Value value; + Value value{}; template bool check(const Key & key) const { return value.first == key; } diff --git a/tests/queries/0_stateless/03009_consecutive_keys_nullable.reference b/tests/queries/0_stateless/03009_consecutive_keys_nullable.reference index 1c8064290c6..e1b9e0cba62 100644 --- a/tests/queries/0_stateless/03009_consecutive_keys_nullable.reference +++ b/tests/queries/0_stateless/03009_consecutive_keys_nullable.reference @@ -1,13 +1,13 @@ -\N 1 1 1 2 0 \N 1 1 1 2 0 +\N 1 1 \N 3 3 1 3 0 -\N 1 1 1 2 0 -\N 2 2 +\N 1 1 1 1 0 +\N 2 2 t_nullable_keys_1 0 t_nullable_keys_2 0 t_nullable_keys_3 1 diff --git a/tests/queries/0_stateless/03009_consecutive_keys_nullable.sql b/tests/queries/0_stateless/03009_consecutive_keys_nullable.sql index ee2cb5a171f..7650cf14a49 100644 --- a/tests/queries/0_stateless/03009_consecutive_keys_nullable.sql +++ b/tests/queries/0_stateless/03009_consecutive_keys_nullable.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS t_nullable_keys_1; CREATE TABLE t_nullable_keys_1 (x Nullable(Int64)) ENGINE = Memory; INSERT INTO t_nullable_keys_1 VALUES (1), (1), (NULL); -SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_1 GROUP BY x; +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_1 GROUP BY x ORDER BY x; DROP TABLE t_nullable_keys_1; @@ -10,7 +10,7 @@ DROP TABLE IF EXISTS t_nullable_keys_2; CREATE TABLE t_nullable_keys_2 (x Nullable(Int64)) ENGINE = Memory; INSERT INTO t_nullable_keys_2 VALUES (NULL), (1), (1); -SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_2 GROUP BY x; +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_2 GROUP BY x ORDER BY x; DROP TABLE t_nullable_keys_2; @@ -18,7 +18,7 @@ DROP TABLE IF EXISTS t_nullable_keys_3; CREATE TABLE t_nullable_keys_3 (x Nullable(Int64)) ENGINE = Memory; INSERT INTO t_nullable_keys_3 VALUES (NULL), (NULL), (NULL); -SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_3 GROUP BY x; +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_3 GROUP BY x ORDER BY x; DROP TABLE t_nullable_keys_3; @@ -26,7 +26,7 @@ DROP TABLE IF EXISTS t_nullable_keys_4; CREATE TABLE t_nullable_keys_4 (x Nullable(Int64)) ENGINE = Memory; INSERT INTO t_nullable_keys_4 VALUES (1), (1), (1); -SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_4 GROUP BY x; +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_4 GROUP BY x ORDER BY x; DROP TABLE t_nullable_keys_4; @@ -34,7 +34,7 @@ DROP TABLE IF EXISTS t_nullable_keys_5; CREATE TABLE t_nullable_keys_5 (x Nullable(Int64)) ENGINE = Memory; INSERT INTO t_nullable_keys_5 VALUES (1), (NULL), (1); -SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_5 GROUP BY x; +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_5 GROUP BY x ORDER BY x; DROP TABLE t_nullable_keys_5; @@ -42,7 +42,7 @@ DROP TABLE IF EXISTS t_nullable_keys_6; CREATE TABLE t_nullable_keys_6 (x Nullable(Int64)) ENGINE = Memory; INSERT INTO t_nullable_keys_6 VALUES (NULL), (1), (NULL); -SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_6 GROUP BY x; +SELECT x, count(), countIf(x IS NULL) FROM t_nullable_keys_6 GROUP BY x ORDER BY x; DROP TABLE t_nullable_keys_6; From 29a52419972ca8bac81d56ced30d871495091cdb Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 20 Mar 2024 17:18:08 +0100 Subject: [PATCH 0365/1165] fix build --- src/Common/Config/AbstractConfigurationComparison.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/Config/AbstractConfigurationComparison.cpp b/src/Common/Config/AbstractConfigurationComparison.cpp index e241a540cc1..607b583cf31 100644 --- a/src/Common/Config/AbstractConfigurationComparison.cpp +++ b/src/Common/Config/AbstractConfigurationComparison.cpp @@ -41,7 +41,7 @@ namespace #if defined(ABORT_ON_LOGICAL_ERROR) /// Compound `ignore_keys` are not yet implemented. for (const auto & ignore_key : *ignore_keys) - chassert(ignore_key.find(".") == std::string_view::npos); + chassert(ignore_key.find('.') == std::string_view::npos); #endif } From 63631ee58721729bd82d45a854074546c38be5f8 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 21 Mar 2024 15:35:05 +0000 Subject: [PATCH 0366/1165] Remove debug code --- src/Storages/StorageMergeTree.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 19f951c8dc4..c87681a1418 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -219,7 +219,6 @@ void StorageMergeTree::read( Block header; if (local_context->getSettingsRef().allow_experimental_analyzer) { - chassert(query_info.query_tree); QueryTreeNodePtr modified_query_tree = query_info.query_tree->clone(); rewriteJoinToGlobalJoin(modified_query_tree, local_context); modified_query_tree = buildQueryTreeForShard(query_info.planner_context, modified_query_tree); From f0491595710a17a43fb313f2f30d74ab9d3fb144 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 21 Mar 2024 15:37:00 +0000 Subject: [PATCH 0367/1165] Try to fix abort in arrow --- contrib/arrow | 2 +- .../Formats/Impl/ArrowBufferedStreams.cpp | 76 +++++++++++++++---- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/contrib/arrow b/contrib/arrow index ba5c67934e8..12232bbbe39 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit ba5c67934e8274d649befcffab56731632dc5253 +Subproject commit 12232bbbe39b4ffbd921a0caff6d046ae009a753 diff --git a/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp b/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp index 83d7a8b7bbd..06819a860d8 100644 --- a/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp +++ b/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp @@ -4,6 +4,7 @@ #if USE_ARROW || USE_ORC || USE_PARQUET #include +#include #include #include #include @@ -11,7 +12,7 @@ #include #include #include -#include +//#include #include @@ -41,9 +42,18 @@ arrow::Result ArrowBufferedOutputStream::Tell() const arrow::Status ArrowBufferedOutputStream::Write(const void * data, int64_t length) { - out.write(reinterpret_cast(data), length); - total_length += length; - return arrow::Status::OK(); + try + { + out.write(reinterpret_cast(data), length); + total_length += length; + return arrow::Status::OK(); + } + catch (...) + { + auto message = getCurrentExceptionMessage(false); + LOG_ERROR(getLogger("ArrowBufferedOutputStream"), "Error while writing to arrow stream: {}", message); + return arrow::Status::IOError(message); + } } RandomAccessFileFromSeekableReadBuffer::RandomAccessFileFromSeekableReadBuffer(ReadBuffer & in_, std::optional file_size_, bool avoid_buffering_) @@ -74,9 +84,18 @@ arrow::Result RandomAccessFileFromSeekableReadBuffer::Tell() const arrow::Result RandomAccessFileFromSeekableReadBuffer::Read(int64_t nbytes, void * out) { - if (avoid_buffering) - in.setReadUntilPosition(seekable_in.getPosition() + nbytes); - return in.readBig(reinterpret_cast(out), nbytes); + try + { + if (avoid_buffering) + in.setReadUntilPosition(seekable_in.getPosition() + nbytes); + return in.readBig(reinterpret_cast(out), nbytes); + } + catch (...) + { + auto message = getCurrentExceptionMessage(false); + LOG_ERROR(getLogger("ArrowBufferedOutputStream"), "Error while reading from arrow stream: {}", message); + return arrow::Status::IOError(message); + } } arrow::Result> RandomAccessFileFromSeekableReadBuffer::Read(int64_t nbytes) @@ -98,14 +117,23 @@ arrow::Future> RandomAccessFileFromSeekableReadBu arrow::Status RandomAccessFileFromSeekableReadBuffer::Seek(int64_t position) { - if (avoid_buffering) + try { - // Seeking to a position above a previous setReadUntilPosition() confuses some of the - // ReadBuffer implementations. - in.setReadUntilEnd(); + if (avoid_buffering) + { + // Seeking to a position above a previous setReadUntilPosition() confuses some of the + // ReadBuffer implementations. + in.setReadUntilEnd(); + } + seekable_in.seek(position, SEEK_SET); + return arrow::Status::OK(); + } + catch (...) + { + auto message = getCurrentExceptionMessage(false); + LOG_ERROR(getLogger("ArrowBufferedOutputStream"), "Error while seeking arrow file: {}", message); + return arrow::Status::IOError(message); } - seekable_in.seek(position, SEEK_SET); - return arrow::Status::OK(); } @@ -115,7 +143,16 @@ ArrowInputStreamFromReadBuffer::ArrowInputStreamFromReadBuffer(ReadBuffer & in_) arrow::Result ArrowInputStreamFromReadBuffer::Read(int64_t nbytes, void * out) { - return in.readBig(reinterpret_cast(out), nbytes); + try + { + return in.readBig(reinterpret_cast(out), nbytes); + } + catch (...) + { + auto message = getCurrentExceptionMessage(false); + LOG_ERROR(getLogger("ArrowBufferedOutputStream"), "Error while reading from arrow stream: {}", message); + return arrow::Status::IOError(message); + } } arrow::Result> ArrowInputStreamFromReadBuffer::Read(int64_t nbytes) @@ -154,7 +191,16 @@ arrow::Result RandomAccessFileFromRandomAccessReadBuffer::GetSize() arrow::Result RandomAccessFileFromRandomAccessReadBuffer::ReadAt(int64_t position, int64_t nbytes, void* out) { - return in.readBigAt(reinterpret_cast(out), nbytes, position); + try + { + return in.readBigAt(reinterpret_cast(out), nbytes, position); + } + catch (...) + { + auto message = getCurrentExceptionMessage(false); + LOG_ERROR(getLogger("ArrowBufferedOutputStream"), "Error while reading from arrow stream: {}", message); + return arrow::Status::IOError(message); + } } arrow::Result> RandomAccessFileFromRandomAccessReadBuffer::ReadAt(int64_t position, int64_t nbytes) From 01fb309d2e925a20e1cae8db8e49c311ffb31f19 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 21 Mar 2024 15:39:15 +0000 Subject: [PATCH 0368/1165] Add test --- tests/queries/0_stateless/02834_apache_arrow_abort.reference | 0 tests/queries/0_stateless/02834_apache_arrow_abort.sql | 4 ++++ 2 files changed, 4 insertions(+) create mode 100644 tests/queries/0_stateless/02834_apache_arrow_abort.reference create mode 100644 tests/queries/0_stateless/02834_apache_arrow_abort.sql diff --git a/tests/queries/0_stateless/02834_apache_arrow_abort.reference b/tests/queries/0_stateless/02834_apache_arrow_abort.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02834_apache_arrow_abort.sql b/tests/queries/0_stateless/02834_apache_arrow_abort.sql new file mode 100644 index 00000000000..47db46f1e43 --- /dev/null +++ b/tests/queries/0_stateless/02834_apache_arrow_abort.sql @@ -0,0 +1,4 @@ +-- Tags: no-fasttest +-- This tests depends on internet access, but it does not matter, because it only has to check that there is no abort due to a bug in Apache Arrow library. + +INSERT INTO TABLE FUNCTION url('https://clickhouse-public-datasets.s3.amazonaws.com/hits_compatible/hits.parquet') SELECT * FROM url('https://clickhouse-public-datasets.s3.amazonaws.com/hits_compatible/hits.parquet'); -- { serverError CANNOT_WRITE_TO_OSTREAM, RECEIVED_ERROR_FROM_REMOTE_IO_SERVER, POCO_EXCEPTION } From 6e8e01d47e100ff92b21151cf2b3e4029625af1f Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 21 Mar 2024 15:46:04 +0000 Subject: [PATCH 0369/1165] Fix headers --- src/Processors/Formats/Impl/ArrowBufferedStreams.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp b/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp index 2fa808ccdf2..84375ccd5ce 100644 --- a/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp +++ b/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp @@ -12,7 +12,7 @@ #include #include #include -//#include +#include #include From 87b1e0ead238e032c100a54697093c87908217f0 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 21 Mar 2024 18:22:11 +0100 Subject: [PATCH 0370/1165] Add more logging --- src/Interpreters/Cache/LRUFileCachePriority.cpp | 1 + src/Interpreters/Cache/LRUFileCachePriority.h | 2 +- src/Interpreters/Cache/SLRUFileCachePriority.cpp | 9 +++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 04622d54a17..ca2127d1c8c 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -37,6 +37,7 @@ LRUFileCachePriority::LRUFileCachePriority( const std::string & description_) : IFileCachePriority(max_size_, max_elements_) , description(description_) + , log(getLogger("LRUFileCachePriority{}" + (description.empty() ? "" : "(" + description + ")"))) { if (state_) state = state_; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index 1f80cfac170..3dea459e271 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -86,7 +86,7 @@ private: LRUQueue queue; const std::string description; - LoggerPtr log = getLogger("LRUFileCachePriority"); + LoggerPtr log; StatePtr state; void updateElementsCount(int64_t num); diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 07d04ee7876..e59a2fcd68f 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -215,7 +215,16 @@ bool SLRUFileCachePriority::collectCandidatesForEviction( hold_space = std::make_shared( hold_size, hold_elements, QueueEntryType::SLRU_Probationary, probationary_queue, lock); + + LOG_TEST(log, "Eviction candidates: {}, hold space: {} in size and {} in elements. {}", + res.size(), hold_size, hold_elements, getStateInfoForLog(lock)); } + else + { + LOG_TEST(log, "Eviction candidates: {}, hold space: none. {}", + res.size(), getStateInfoForLog(lock)); + } + auto downgrade_func = [=, this] (const CachePriorityGuard::Lock & lk) From 110b08f666c158d0f0cabbeff975e56b57f2286f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 18:38:57 +0100 Subject: [PATCH 0371/1165] Fixup #ci_set_analyzer --- src/Planner/PlannerExpressionAnalysis.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index 441e2ac4b9e..dd3769ee10b 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -113,7 +113,7 @@ std::optional analyzeAggregation(const QueryTreeNodeP continue; auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; - auto column_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; + auto column_after_aggregation = group_by_use_nulls && expression_dag_node->column != nullptr ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); @@ -164,7 +164,7 @@ std::optional analyzeAggregation(const QueryTreeNodeP continue; auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; - auto column_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; + auto column_after_aggregation = group_by_use_nulls && expression_dag_node->column != nullptr ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); From b275e06aca70a334babf74412fea6cf728664bca Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 19:02:46 +0100 Subject: [PATCH 0372/1165] Add test for 57820 #ci_set_analyzer --- .../queries/0_stateless/03018_analyzer_greater_null.reference | 1 + tests/queries/0_stateless/03018_analyzer_greater_null.sql | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 tests/queries/0_stateless/03018_analyzer_greater_null.reference create mode 100644 tests/queries/0_stateless/03018_analyzer_greater_null.sql diff --git a/tests/queries/0_stateless/03018_analyzer_greater_null.reference b/tests/queries/0_stateless/03018_analyzer_greater_null.reference new file mode 100644 index 00000000000..1e254619239 --- /dev/null +++ b/tests/queries/0_stateless/03018_analyzer_greater_null.reference @@ -0,0 +1 @@ +\N 0 diff --git a/tests/queries/0_stateless/03018_analyzer_greater_null.sql b/tests/queries/0_stateless/03018_analyzer_greater_null.sql new file mode 100644 index 00000000000..66ea53e451f --- /dev/null +++ b/tests/queries/0_stateless/03018_analyzer_greater_null.sql @@ -0,0 +1,4 @@ +SELECT + max(NULL > 255) > NULL AS a, + count(NULL > 1.) > 1048577 +FROM numbers(10); From 958659584957ff419a9305d9c7edee5703fedbdc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 19:09:40 +0100 Subject: [PATCH 0373/1165] Fix clang-tidy --- src/Functions/FunctionsTimeWindow.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index 65cc7e9e87c..d52b76bec91 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -119,7 +119,7 @@ template<> \ struct AddTime \ { \ static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) \ - { return static_cast(t + delta * INTERVAL); } \ + { return static_cast(t + delta * (INTERVAL)); } \ }; ADD_TIME(Day, 86400) ADD_TIME(Hour, 3600) From 89d80a4d65be42986f8d710c6f2ac5305491b80b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 Mar 2024 18:16:31 +0000 Subject: [PATCH 0374/1165] Redo fix. --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 57 ++++++++++++++++++----- src/Functions/randConstant.cpp | 16 ++----- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 2a46417ee98..834bc2e5813 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -799,7 +799,13 @@ struct IdentifierResolveScope /// Node hash to mask id map std::shared_ptr> projection_mask_map; - std::map rand_constant_cache; + struct ResolvedFunctionsCache + { + FunctionOverloadResolverPtr resolver; + std::map cache; + }; + + std::map functions_cache; [[maybe_unused]] const IdentifierResolveScope * getNearestQueryScope() const { @@ -925,6 +931,24 @@ struct IdentifierResolveScope } }; +IQueryTreeNode::Hash getHashForFunctionArguments(const ColumnsWithTypeAndName & arguments) +{ + SipHash hash; + for (const auto & arg : arguments) + { + auto type_name = arg.type->getName(); + hash.update(type_name.c_str(), type_name.size()); + + if (arg.column) + { + if (const auto * col_const = typeid_cast(arg.column.get())) + col_const->updateHashWithValue(0, hash); + } + } + + return getSipHash128AsPair(hash); +} + /** Visitor that extracts expression and function aliases from node and initialize scope tables with it. * Does not go into child lambdas and queries. @@ -5534,21 +5558,19 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi FunctionOverloadResolverPtr function = UserDefinedExecutableFunctionFactory::instance().tryGet(function_name, scope.context, parameters); bool is_executable_udf = true; + IdentifierResolveScope::ResolvedFunctionsCache * function_cache = nullptr; + if (!function) { /// This is a hack to allow a query like `select randConstant(), randConstant(), randConstant()`. /// Function randConstant() would return the same value for the same arguments (in scope). - if (function_name == "randConstant") - { - auto hash = function_node_ptr->getTreeHash(); - auto & func = scope.rand_constant_cache[hash]; - if (!func) - func = FunctionFactory::instance().tryGet(function_name, scope.context); - function = func; - } - else - function = FunctionFactory::instance().tryGet(function_name, scope.context); + auto hash = function_node_ptr->getTreeHash(); + function_cache = &scope.functions_cache[hash]; + if (!function_cache->resolver) + function_cache->resolver = FunctionFactory::instance().tryGet(function_name, scope.context); + + function = function_cache->resolver; is_executable_udf = false; } @@ -5773,7 +5795,18 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi try { - auto function_base = function->build(argument_columns); + FunctionBasePtr function_base; + if (function_cache) + { + auto args_hash = getHashForFunctionArguments(argument_columns); + auto & cached_function = function_cache->cache[args_hash]; + if (!cached_function) + cached_function = function->build(argument_columns); + + function_base = cached_function; + } + else + function_base = function->build(argument_columns); /// Do not constant fold get scalar functions bool disable_constant_folding = function_name == "__getScalar" || function_name == "shardNum" || diff --git a/src/Functions/randConstant.cpp b/src/Functions/randConstant.cpp index 5872ab8bb34..22ce6d88ea6 100644 --- a/src/Functions/randConstant.cpp +++ b/src/Functions/randConstant.cpp @@ -88,14 +88,6 @@ public: return std::make_unique>(); } - RandomConstantOverloadResolver() - { - typename ColumnVector::Container vec_to(1); - - TargetSpecific::Default::RandImpl::execute(reinterpret_cast(vec_to.data()), sizeof(ToType)); - value = vec_to[0]; - } - DataTypePtr getReturnTypeImpl(const DataTypes & data_types) const override { size_t number_of_arguments = data_types.size(); @@ -113,11 +105,13 @@ public: if (!arguments.empty()) argument_types.emplace_back(arguments.back().type); + typename ColumnVector::Container vec_to(1); + + TargetSpecific::Default::RandImpl::execute(reinterpret_cast(vec_to.data()), sizeof(ToType)); + ToType value = vec_to[0]; + return std::make_unique>(value, argument_types, return_type); } - -private: - ToType value; }; struct NameRandConstant { static constexpr auto name = "randConstant"; }; From 929173cce2e934a5510a0b9753d92734f39bf8d1 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 21 Mar 2024 19:18:16 +0100 Subject: [PATCH 0375/1165] Fix S3 buffer allocation --- src/Common/BufferAllocationPolicy.cpp | 2 +- src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp | 4 ++++ src/IO/WriteBufferFromS3.cpp | 12 ++++++++++++ src/IO/WriteBufferFromS3.h | 1 + 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Common/BufferAllocationPolicy.cpp b/src/Common/BufferAllocationPolicy.cpp index 980cbcca729..9f9665be9b9 100644 --- a/src/Common/BufferAllocationPolicy.cpp +++ b/src/Common/BufferAllocationPolicy.cpp @@ -73,7 +73,7 @@ public: if (1 == buffer_number) { - current_size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), first_size); + current_size = first_size; return; } diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index ed3ce76bd94..1e589cf8bf3 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -141,6 +141,10 @@ void WriteBufferFromAzureBlobStorage::allocateBuffer() { buffer_allocation_policy->nextBuffer(); auto size = buffer_allocation_policy->getBufferSize(); + + if (buffer_allocation_policy->getBufferNumber() == 1) + size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), size); + memory = Memory(size); WriteBuffer::set(memory.data(), memory.size()); } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 865bac86ff5..4ddff4ce898 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -339,10 +339,22 @@ void WriteBufferFromS3::allocateBuffer() { buffer_allocation_policy->nextBuffer(); chassert(0 == hidden_size); + + if (buffer_allocation_policy->getBufferNumber() == 1) + return allocateFirstBuffer(); + memory = Memory(buffer_allocation_policy->getBufferSize()); WriteBuffer::set(memory.data(), memory.size()); } +void WriteBufferFromS3::allocateFirstBuffer() +{ + const auto max_first_buffer = buffer_allocation_policy->getBufferSize(); + const auto size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), max_first_buffer); + memory = Memory(size); + WriteBuffer::set(memory.data(), memory.size()); +} + void WriteBufferFromS3::setFakeBufferWhenPreFinalized() { WriteBuffer::set(fake_buffer_when_prefinalized, sizeof(fake_buffer_when_prefinalized)); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 0eab20891bf..4139cbdde6e 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -62,6 +62,7 @@ private: void reallocateFirstBuffer(); void detachBuffer(); void allocateBuffer(); + void allocateFirstBuffer(); void setFakeBufferWhenPreFinalized(); S3::UploadPartRequest getUploadRequest(size_t part_number, PartData & data); From 6a19b77d9ea4545bf7080896a760022ed8fd9410 Mon Sep 17 00:00:00 2001 From: yariks5s Date: Thu, 21 Mar 2024 18:39:05 +0000 Subject: [PATCH 0376/1165] init --- .../settings.md | 4 +- docs/en/sql-reference/statements/undrop.md | 48 +++++++------------ src/Interpreters/DatabaseCatalog.cpp | 4 +- src/Interpreters/InterpreterUndropQuery.cpp | 6 ++- tests/integration/test_undrop_query/test.py | 42 ++++++++++++---- .../0_stateless/02681_undrop_query.sql | 2 +- 6 files changed, 60 insertions(+), 46 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 07c9a2b88ab..f20dcb9025e 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -933,9 +933,9 @@ Hard limit is configured via system tools ## database_atomic_delay_before_drop_table_sec {#database_atomic_delay_before_drop_table_sec} -Sets the delay before remove table data in seconds. If the query has `SYNC` modifier, this setting is ignored. +The delay before a table data is dropped in seconds. If the `DROP TABLE` query has a `SYNC` modifier, this setting is ignored. -Default value: `480` (8 minute). +Default value: `480` (8 minutes). ## database_catalog_unused_dir_hide_timeout_sec {#database_catalog_unused_dir_hide_timeout_sec} diff --git a/docs/en/sql-reference/statements/undrop.md b/docs/en/sql-reference/statements/undrop.md index 40ac1ab4f99..4b138bfe679 100644 --- a/docs/en/sql-reference/statements/undrop.md +++ b/docs/en/sql-reference/statements/undrop.md @@ -13,13 +13,6 @@ a system table called `system.dropped_tables`. If you have a materialized view without a `TO` clause associated with the dropped table, then you will also have to UNDROP the inner table of that view. -:::note -UNDROP TABLE is experimental. To use it add this setting: -```sql -set allow_experimental_undrop_table_query = 1; -``` -::: - :::tip Also see [DROP TABLE](/docs/en/sql-reference/statements/drop.md) ::: @@ -32,60 +25,53 @@ UNDROP TABLE [db.]name [UUID ''] [ON CLUSTER cluster] **Example** -``` sql -set allow_experimental_undrop_table_query = 1; -``` - ```sql -CREATE TABLE undropMe +CREATE TABLE tab ( `id` UInt8 ) ENGINE = MergeTree -ORDER BY id -``` +ORDER BY id; + +DROP TABLE tab; -```sql -DROP TABLE undropMe -``` -```sql SELECT * FROM system.dropped_tables -FORMAT Vertical +FORMAT Vertical; ``` + ```response Row 1: ────── index: 0 database: default -table: undropMe +table: tab uuid: aa696a1a-1d70-4e60-a841-4c80827706cc engine: MergeTree -metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.undropMe.aa696a1a-1d70-4e60-a841-4c80827706cc.sql +metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.tab.aa696a1a-1d70-4e60-a841-4c80827706cc.sql table_dropped_time: 2023-04-05 14:12:12 1 row in set. Elapsed: 0.001 sec. ``` + ```sql -UNDROP TABLE undropMe -``` -```response -Ok. -``` -```sql +UNDROP TABLE tab; + SELECT * FROM system.dropped_tables -FORMAT Vertical -``` +FORMAT Vertical; + ```response Ok. 0 rows in set. Elapsed: 0.001 sec. ``` + ```sql -DESCRIBE TABLE undropMe -FORMAT Vertical +DESCRIBE TABLE tab +FORMAT Vertical; ``` + ```response Row 1: ────── diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 3dc700e9f93..ec6c8b5924f 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -1143,7 +1143,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id) TableMarkedAsDropped dropped_table; { std::lock_guard lock(tables_marked_dropped_mutex); - time_t latest_drop_time = std::numeric_limits::min(); + auto latest_drop_time = std::numeric_limits::min(); auto it_dropped_table = tables_marked_dropped.end(); for (auto it = tables_marked_dropped.begin(); it != tables_marked_dropped.end(); ++it) { @@ -1168,7 +1168,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id) } if (it_dropped_table == tables_marked_dropped.end()) throw Exception(ErrorCodes::UNKNOWN_TABLE, - "The drop task of table {} is in progress, has been dropped or the database engine doesn't support it", + "Table {} is being dropped, has been dropped, or the database engine does not support UNDROP", table_id.getNameForLogs()); latest_metadata_dropped_path = it_dropped_table->metadata_path; String table_metadata_path = getPathForMetadata(it_dropped_table->table_id); diff --git a/src/Interpreters/InterpreterUndropQuery.cpp b/src/Interpreters/InterpreterUndropQuery.cpp index 1f2ff4b4461..920df3d6aed 100644 --- a/src/Interpreters/InterpreterUndropQuery.cpp +++ b/src/Interpreters/InterpreterUndropQuery.cpp @@ -18,14 +18,16 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; } -InterpreterUndropQuery::InterpreterUndropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) +InterpreterUndropQuery::InterpreterUndropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_) + , query_ptr(query_ptr_) { } - BlockIO InterpreterUndropQuery::execute() { getContext()->checkAccess(AccessType::UNDROP_TABLE); + auto & undrop = query_ptr->as(); if (!undrop.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext())) { diff --git a/tests/integration/test_undrop_query/test.py b/tests/integration/test_undrop_query/test.py index 590a5690e55..2aa5bdacf9d 100644 --- a/tests/integration/test_undrop_query/test.py +++ b/tests/integration/test_undrop_query/test.py @@ -29,30 +29,56 @@ def test_undrop_drop_and_undrop_loop(started_cluster): logging.info( "random_sec: " + random_sec.__str__() + ", table_uuid: " + table_uuid ) + node.query( - "create table test_undrop_loop" + "CREATE TABLE test_undrop_loop" + count.__str__() + " UUID '" + table_uuid - + "' (id Int32) Engine=MergeTree() order by id;" + + "' (id Int32) ENGINE = MergeTree() ORDER BY id;" ) - node.query("drop table test_undrop_loop" + count.__str__() + ";") + + node.query("DROP TABLE test_undrop_loop" + count.__str__() + ";") + time.sleep(random_sec) + if random_sec >= 5: error = node.query_and_get_error( - "undrop table test_undrop_loop" + "UNDROP TABLE test_undrop_loop" + count.__str__() - + " uuid '" + + " UUID '" + table_uuid + "';" ) assert "UNKNOWN_TABLE" in error - else: + elif random_sec <= 3: + # (*) node.query( - "undrop table test_undrop_loop" + "UNDROP TABLE test_undrop_loop" + count.__str__() - + " uuid '" + + " UUID '" + table_uuid + "';" ) count = count + 1 + else: + try: + node.query( + "UNDROP TABLE test_undrop_loop" + + count.__str__() + + " UUID '" + + table_uuid + + "';" + ) + count = count + 1 + except: + error = node.query_and_get_error( + "UNDROP TABLE test_undrop_loop" + + count.__str__() + + " UUID '" + + table_uuid + + "';" + ) + assert "UNKNOWN_TABLE" in error + # ignore random_sec = 4 result to account for communication delay with the database. + # if we don't do that, then the second case (*) may find the table already dropped and receive an unexpected exception from the database (Bug #55167) diff --git a/tests/queries/0_stateless/02681_undrop_query.sql b/tests/queries/0_stateless/02681_undrop_query.sql index 66447fc6c44..d038a383690 100644 --- a/tests/queries/0_stateless/02681_undrop_query.sql +++ b/tests/queries/0_stateless/02681_undrop_query.sql @@ -85,5 +85,5 @@ drop table 02681_undrop_multiple; select table from system.dropped_tables where table = '02681_undrop_multiple' limit 1; undrop table 02681_undrop_multiple; select * from 02681_undrop_multiple order by id; -undrop table 02681_undrop_multiple; -- { serverError 57 } +undrop table 02681_undrop_multiple; -- { serverError TABLE_ALREADY_EXISTS } drop table 02681_undrop_multiple sync; From 2658cca513b43a910f570f8adf957f2c63f143a8 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 21 Mar 2024 19:42:46 +0100 Subject: [PATCH 0377/1165] Update 02916_move_partition_inactive_replica.sql --- .../0_stateless/02916_move_partition_inactive_replica.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql index d597c037612..25dd0365d97 100644 --- a/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql +++ b/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql @@ -32,7 +32,7 @@ drop table if exists shard_0.from_1; drop table if exists shard_1.from_1; OPTIMIZE TABLE shard_0.to; OPTIMIZE TABLE shard_0.to; -select name, active from system.parts where database='shard_0' and table='to' order by name; +select name, active from system.parts where database='shard_0' and table='to' and active order by name; system restart replica shard_0.to; From 3745df45900dd89348baa46b10ec7c55967d548c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 21 Mar 2024 19:43:12 +0100 Subject: [PATCH 0378/1165] Update 02916_move_partition_inactive_replica.reference --- .../02916_move_partition_inactive_replica.reference | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/queries/0_stateless/02916_move_partition_inactive_replica.reference b/tests/queries/0_stateless/02916_move_partition_inactive_replica.reference index b64cc6289d5..361a0d1e9bb 100644 --- a/tests/queries/0_stateless/02916_move_partition_inactive_replica.reference +++ b/tests/queries/0_stateless/02916_move_partition_inactive_replica.reference @@ -1,8 +1,4 @@ -all_0_0_0 0 all_0_1_1 1 -all_1_1_0 0 -all_2_2_0 0 all_2_3_1 1 -all_3_3_0 0 0 40 1580 From 3f653bbc3258793be95c94ce749142b8b6e0f4db Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 21 Mar 2024 20:28:52 +0100 Subject: [PATCH 0379/1165] Added test with log check --- .../IO/ReadBufferFromAzureBlobStorage.cpp | 18 ++++--- .../test_storage_azure_blob_storage/test.py | 23 +++++++++ ...xx_test_dictionary_create_access.reference | 0 .../xxx_test_dictionary_create_access.sh | 47 +++++++++++++++++++ 4 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/xxx_test_dictionary_create_access.reference create mode 100755 tests/queries/0_stateless/xxx_test_dictionary_create_access.sh diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 3555c71abee..0c4acebf347 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -264,15 +264,6 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran size_t initial_n = n; size_t sleep_time_with_backoff_milliseconds = 100; - auto handle_exception = [&, this](const auto & e, size_t i) - { - LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message); - if (i + 1 == max_single_download_retries) - throw; - - sleepForMilliseconds(sleep_time_with_backoff_milliseconds); - sleep_time_with_backoff_milliseconds *= 2; - }; for (size_t i = 0; i < max_single_download_retries && n > 0; ++i) { @@ -288,12 +279,19 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran std::istringstream string_stream(String(static_cast(data_ptr),bytes)); // STYLE_CHECK_ALLOW_STD_STRING_STREAM copyFromIStreamWithProgressCallback(string_stream, to, n, progress_callback, &bytes_copied); + LOG_INFO(log, "AzureBlobStorage readBigAt read bytes {}", bytes_copied); + if (read_settings.remote_throttler) read_settings.remote_throttler->add(bytes_copied, ProfileEvents::RemoteReadThrottlerBytes, ProfileEvents::RemoteReadThrottlerSleepMicroseconds); } catch (const Azure::Core::RequestFailedException & e) { - handle_exception(e,i); + LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message); + if (i + 1 == max_single_download_retries) + throw; + + sleepForMilliseconds(sleep_time_with_backoff_milliseconds); + sleep_time_with_backoff_milliseconds *= 2; } range_begin += bytes_copied; diff --git a/tests/integration/test_storage_azure_blob_storage/test.py b/tests/integration/test_storage_azure_blob_storage/test.py index e1d636f3831..6918be9f78a 100644 --- a/tests/integration/test_storage_azure_blob_storage/test.py +++ b/tests/integration/test_storage_azure_blob_storage/test.py @@ -16,6 +16,7 @@ from helpers.cluster import ClickHouseCluster, ClickHouseInstance from helpers.network import PartitionManager from helpers.mock_servers import start_mock_servers from helpers.test_tools import exec_query_with_retry +from helpers.test_tools import assert_logs_contain_with_retry @pytest.fixture(scope="module") @@ -1320,3 +1321,25 @@ def test_format_detection(cluster): ) assert result == expected_result + + +def test_parallel_read(cluster): + node = cluster.instances["node"] + connection_string = cluster.env_variables["AZURITE_CONNECTION_STRING"] + storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] + account_name = "devstoreaccount1" + account_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + + azure_query( + node, + f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_parallel_read.parquet', '{account_name}', '{account_key}') " + f"select * from numbers(10000) settings azure_truncate_on_insert=1", + ) + time.sleep(1) + + res = azure_query( + node, + f"select count() from azureBlobStorage('{connection_string}', 'cont', 'test_parallel_read.parquet')", + ) + assert int(res) == 10000 + assert_logs_contain_with_retry(node, "AzureBlobStorage readBigAt read bytes") \ No newline at end of file diff --git a/tests/queries/0_stateless/xxx_test_dictionary_create_access.reference b/tests/queries/0_stateless/xxx_test_dictionary_create_access.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/xxx_test_dictionary_create_access.sh b/tests/queries/0_stateless/xxx_test_dictionary_create_access.sh new file mode 100755 index 00000000000..4478eb59c95 --- /dev/null +++ b/tests/queries/0_stateless/xxx_test_dictionary_create_access.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +readonly PID=$$ +readonly TEST_USER=$"02834_USER_${PID}" + +#${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \ +# -d "SELECT * FROM remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', 'system', 'one', '${TEST_USER}', 'pass')" + +${CLICKHOUSE_CLIENT} -q "CREATE DATABASE db;" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db.table;" + +${CLICKHOUSE_CLIENT} -q "DROP DICTIONARY IF EXISTS db.dict_name;" + +${CLICKHOUSE_CLIENT} -q "CREATE TABLE db.table (label_data_combination_id UInt64) engine = MergeTree Order BY label_data_combination_id;" +${CLICKHOUSE_CLIENT} -q "CREATE USER ${TEST_USER};" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON db.* TO ${TEST_USER};" +#${CLICKHOUSE_CLIENT} -q "GRANT INSERT ON db.* TO ${TEST_USER};" +${CLICKHOUSE_CLIENT} -q "GRANT CREATE ON db.* TO ${TEST_USER};" +#${CLICKHOUSE_CLIENT} -q "GRANT DROP ON db.* TO ${TEST_USER};" + +${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \ + -d "CREATE OR REPLACE DICTIONARY db.dict_name + ( + label_data_combination_id UInt64 + ) + PRIMARY KEY label_data_combination_id + SOURCE(CLICKHOUSE( + QUERY + 'SELECT + label_data_combination_id + FROM db.table' + ) + ) + LAYOUT(HASHED(SHARDS 16 SHARD_LOAD_QUEUE_BACKLOG 10000)) + LIFETIME(0) + SETTINGS(dictionary_use_async_executor = 1, max_threads = 16) + COMMENT 'Dictionary mapping of label_data_combination_id to the underlying data.';" + + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db.table;" +${CLICKHOUSE_CLIENT} -q "DROP DICTIONARY IF EXISTS db.dict_name;" +${CLICKHOUSE_CLIENT} -q "DROP DATABASE db;" From d6b515b6bf6069b980509bfa0bd4f8fdcd088acf Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Thu, 21 Mar 2024 17:20:08 +0000 Subject: [PATCH 0380/1165] CI: modify CI from PR body #do_not_test #job_style_check --- .github/PULL_REQUEST_TEMPLATE.md | 30 ++++++++++++++++++++++++++++++ .gitmessage | 2 +- tests/ci/ci.py | 21 +++++++++++++++------ tests/ci/pr_info.py | 3 +++ 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7fb2abebbbb..4db0737c959 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -40,3 +40,33 @@ At a minimum, the following information should be added (but add more as needed) > Information about CI checks: https://clickhouse.com/docs/en/development/continuous-integration/ + +--- +### Modify your CI run: +##### NOTE: +- if your merge the PR with modified CI you **MUST** know what you are doing. +- modifiers can be applied only if set before CI starts +- remove `!` to apply +- return all `!` to restore defaults +``` +!#ci_set_ - to run only preconfigured set of tests, e.g.: +!#ci_set_arm - to run only integration tests on ARM +!#ci_set_integration - to run only integration tests on AMD +!#ci_set_analyzer - to run only tests for analyzer +NOTE: you can configure your own ci set +``` +``` +!#job_ - to run only specified job, e.g.: +!#job_stateless_tests_release +!#job_package_debug +!#job_style_check +!#job_integration_tests_asan +``` +``` +!#batch_2 - to run only 2nd batch for all multi-batch jobs +!#btach_1_2_3 - to run only 1, 2, 3rd batch for all multi-batch jobs +``` +``` +!#no_merge_commit - to disable merge commit (no merge from master) +!#do_not_test - to disable whole CI (except style check) +``` diff --git a/.gitmessage b/.gitmessage index 2ad30596de6..797446edd49 100644 --- a/.gitmessage +++ b/.gitmessage @@ -26,4 +26,4 @@ ## To run only specified batches for multi-batch job(s) #batch_2 -#btach_1_2_3 +#batch_1_2_3 diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 29906e6571f..cd63514cb6a 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1407,15 +1407,25 @@ def _update_gh_statuses_action(indata: Dict, s3: S3Helper) -> None: print("... CI report update - done") -def _fetch_commit_tokens(message: str) -> List[str]: - pattern = r"#[\w-]+" - matches = [match[1:] for match in re.findall(pattern, message)] +def _fetch_commit_tokens(message: str, pr_info: PRInfo) -> List[str]: + pattern = r"([^!]|^)#(\w+)" + matches = [match[-1] for match in re.findall(pattern, message)] res = [ match for match in matches if match in Labels or match.startswith("job_") or match.startswith("batch_") ] - return res + print(f"CI modifyers from commit message: [{res}]") + res_2 = [] + if pr_info.is_pr(): + matches = [match[-1] for match in re.findall(pattern, pr_info.body)] + res_2 = [ + match + for match in matches + if match in Labels or match.startswith("job_") or match.startswith("batch_") + ] + print(f"CI modifyers from PR body: [{res_2}]") + return list(set(res + res_2)) def _upload_build_artifacts( @@ -1701,8 +1711,7 @@ def main() -> int: message = args.commit_message or git_runner.run( f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1" ) - tokens = _fetch_commit_tokens(message) - print(f"Commit message tokens: [{tokens}]") + tokens = _fetch_commit_tokens(message, pr_info) if Labels.NO_MERGE_COMMIT in tokens and CI: git_runner.run(f"{GIT_PREFIX} checkout {pr_info.sha}") git_ref = git_runner.run(f"{GIT_PREFIX} rev-parse HEAD") diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 9bd30f3c58e..84f2db4002d 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -316,6 +316,9 @@ class PRInfo: def is_release_branch(self) -> bool: return self.number == 0 + def is_pr(self): + return self.event_type == EventType.PULL_REQUEST + def is_scheduled(self): return self.event_type == EventType.SCHEDULE From 54c5ea8d6d1e13b8151bbfcc7cffd8f1de5c616b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 21 Mar 2024 21:14:24 +0100 Subject: [PATCH 0381/1165] FunctionsConversion: Remove some templates --- src/Functions/FunctionsConversion.cpp | 164 ++++++++++++-------------- 1 file changed, 78 insertions(+), 86 deletions(-) diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index ceff4f3fd7e..9262389e593 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -1127,23 +1127,24 @@ struct AccurateOrNullConvertStrategyAdditions UInt32 scale { 0 }; }; - -struct ConvertDefaultBehaviorTag {}; -struct ConvertReturnNullOnErrorTag {}; -struct ConvertReturnZeroOnErrorTag {}; +enum class BehaviourOnErrorFromString +{ + ConvertDefaultBehaviorTag, + ConvertReturnNullOnErrorTag, + ConvertReturnZeroOnErrorTag +}; /** Conversion of number types to each other, enums to numbers, dates and datetimes to numbers and back: done by straight assignment. * (Date is represented internally as number of days from some day; DateTime - as unix timestamp) */ template struct ConvertImpl { template static ColumnPtr NO_SANITIZE_UNDEFINED execute( const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type [[maybe_unused]], size_t input_rows_count, - Additions additions = Additions()) + BehaviourOnErrorFromString from_string_tag [[maybe_unused]], Additions additions = Additions()) { const ColumnWithTypeAndName & named_from = arguments[0]; @@ -1176,8 +1177,7 @@ struct ConvertImpl return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); } - else if constexpr (std::is_same_v && std::is_same_v - && std::is_same_v) + else if constexpr (std::is_same_v && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count, additions); @@ -1194,8 +1194,7 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1203,8 +1202,7 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1214,8 +1212,7 @@ struct ConvertImpl || std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1223,8 +1220,7 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1232,8 +1228,7 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1243,8 +1238,7 @@ struct ConvertImpl || std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1254,15 +1248,13 @@ struct ConvertImpl std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); } else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1271,8 +1263,7 @@ struct ConvertImpl std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1282,15 +1273,13 @@ struct ConvertImpl || std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count, additions); } else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count, additions); @@ -1298,23 +1287,20 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count, additions); } /// Conversion of DateTime64 to Date or DateTime: discards fractional part. else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl>, false>::template execute( arguments, result_type, input_rows_count, additions); } else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl>, false>::template execute( arguments, result_type, input_rows_count, additions); @@ -1324,8 +1310,7 @@ struct ConvertImpl std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl::template execute( arguments, result_type, input_rows_count, additions); @@ -1443,8 +1428,7 @@ struct ConvertImpl /// Conversion from FixedString to String. /// Cutting sequences of zero bytes from end of strings. else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { ColumnUInt8::MutablePtr null_map = copyNullMap(arguments[0].column); const auto & nested = columnGetNested(arguments[0]); @@ -1488,8 +1472,7 @@ struct ConvertImpl throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", arguments[0].column->getName(), Name::name); } - else if constexpr (std::is_same_v - && std::is_same_v) + else if constexpr (std::is_same_v) { /// Anything else to String. @@ -1550,27 +1533,35 @@ struct ConvertImpl && std::is_same_v && std::is_same_v) { - return ConvertImpl::template execute( - arguments, result_type, input_rows_count); + return ConvertImpl::template execute( + arguments, result_type, input_rows_count, from_string_tag); } - else if constexpr ((std::is_same_v || std::is_same_v) - && std::is_same_v) + else if constexpr ((std::is_same_v || std::is_same_v)) { - return ConvertThroughParsing::execute( - arguments, result_type, input_rows_count, additions); - } - else if constexpr ((std::is_same_v || std::is_same_v) - && std::is_same_v) - { - return ConvertThroughParsing::execute( - arguments, result_type, input_rows_count, additions); - } - else if constexpr ((std::is_same_v || std::is_same_v) - && is_any_of - && std::is_same_v) - { - return ConvertThroughParsing::execute( - arguments, result_type, input_rows_count, additions); + switch (from_string_tag) + { + case BehaviourOnErrorFromString::ConvertDefaultBehaviorTag: + return ConvertThroughParsing::execute( + arguments, result_type, input_rows_count, additions); + case BehaviourOnErrorFromString::ConvertReturnNullOnErrorTag: + return ConvertThroughParsing::execute( + arguments, result_type, input_rows_count, additions); + case BehaviourOnErrorFromString::ConvertReturnZeroOnErrorTag: + return ConvertThroughParsing::execute( + arguments, result_type, input_rows_count, additions); + } } else { @@ -2165,12 +2156,11 @@ private: if (context) date_time_overflow_behavior = context->getSettingsRef().date_time_overflow_behavior.value; - auto call = [&](const auto & types, const auto & tag) -> bool + auto call = [&](const auto & types, BehaviourOnErrorFromString from_string_tag) -> bool { using Types = std::decay_t; using LeftDataType = typename Types::LeftType; using RightDataType = typename Types::RightType; - using SpecialTag = std::decay_t; if constexpr (IsDataTypeDecimal) { @@ -2191,13 +2181,13 @@ private: switch (date_time_overflow_behavior) { case FormatSettings::DateTimeOverflowBehavior::Throw: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, scale); break; case FormatSettings::DateTimeOverflowBehavior::Ignore: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, scale); break; case FormatSettings::DateTimeOverflowBehavior::Saturate: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, scale); break; } @@ -2208,20 +2198,20 @@ private: switch (date_time_overflow_behavior) { case FormatSettings::DateTimeOverflowBehavior::Throw: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, dt64->getScale()); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, dt64->getScale()); break; case FormatSettings::DateTimeOverflowBehavior::Ignore: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, dt64->getScale()); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, dt64->getScale()); break; case FormatSettings::DateTimeOverflowBehavior::Saturate: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, dt64->getScale()); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, dt64->getScale()); break; } } #define GENERATE_OVERFLOW_MODE_CASE(OVERFLOW_MODE) \ case FormatSettings::DateTimeOverflowBehavior::OVERFLOW_MODE: \ - result_column = ConvertImpl::execute( \ - arguments, result_type, input_rows_count); \ + result_column = ConvertImpl::execute( \ + arguments, result_type, input_rows_count, from_string_tag); \ break; else if constexpr (IsDataTypeDecimalOrNumber && IsDataTypeDecimalOrNumber) @@ -2262,7 +2252,7 @@ private: } #undef GENERATE_OVERFLOW_MODE_CASE else - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag); return true; }; @@ -2275,7 +2265,7 @@ private: if (to_datetime64 || scale != 0) /// When scale = 0, the data type is DateTime otherwise the data type is DateTime64 { - if (!callOnIndexAndDataType(from_type->getTypeId(), call, ConvertDefaultBehaviorTag{})) + if (!callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0].type->getName(), getName()); @@ -2292,23 +2282,25 @@ private: bool done = false; if constexpr (is_any_of) { - done = callOnIndexAndDataType(from_type->getTypeId(), call, ConvertDefaultBehaviorTag{}); + done = callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag); } else { bool cast_ipv4_ipv6_default_on_conversion_error = false; if constexpr (is_any_of) + { if (context && (cast_ipv4_ipv6_default_on_conversion_error = context->getSettingsRef().cast_ipv4_ipv6_default_on_conversion_error)) - done = callOnIndexAndDataType(from_type->getTypeId(), call, ConvertReturnZeroOnErrorTag{}); + done = callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertReturnZeroOnErrorTag); + } if (!cast_ipv4_ipv6_default_on_conversion_error) { /// We should use ConvertFromStringExceptionMode::Null mode when converting from String (or FixedString) /// to Nullable type, to avoid 'value is too short' error on attempt to parse empty string from NULL values. if (to_nullable && WhichDataType(from_type).isStringOrFixedString()) - done = callOnIndexAndDataType(from_type->getTypeId(), call, ConvertReturnNullOnErrorTag{}); + done = callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertReturnNullOnErrorTag); else - done = callOnIndexAndDataType(from_type->getTypeId(), call, ConvertDefaultBehaviorTag{}); + done = callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag); } } @@ -3185,8 +3177,8 @@ private: { #define GENERATE_OVERFLOW_MODE_CASE(OVERFLOW_MODE, ADDITIONS) \ case FormatSettings::DateTimeOverflowBehavior::OVERFLOW_MODE: \ - result_column = ConvertImpl::execute( \ - arguments, result_type, input_rows_count, ADDITIONS()); \ + result_column = ConvertImpl::execute( \ + arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag, ADDITIONS()); \ break; if (wrapper_cast_type == CastType::accurate) { @@ -3215,8 +3207,8 @@ private: { #define GENERATE_OVERFLOW_MODE_CASE(OVERFLOW_MODE, ADDITIONS) \ case FormatSettings::DateTimeOverflowBehavior::OVERFLOW_MODE: \ - result_column = ConvertImpl::template execute( \ -arguments, result_type, input_rows_count); \ + result_column = ConvertImpl::template execute( \ +arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag); \ break; if (wrapper_cast_type == CastType::accurate) { @@ -3376,7 +3368,7 @@ arguments, result_type, input_rows_count); \ AccurateConvertStrategyAdditions additions; additions.scale = scale; result_column = ConvertImpl::execute( - arguments, result_type, input_rows_count, additions); + arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag, additions); return true; } @@ -3385,7 +3377,7 @@ arguments, result_type, input_rows_count); \ AccurateOrNullConvertStrategyAdditions additions; additions.scale = scale; result_column = ConvertImpl::execute( - arguments, result_type, input_rows_count, additions); + arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag, additions); return true; } @@ -3397,14 +3389,14 @@ arguments, result_type, input_rows_count); \ /// Consistent with CAST(Nullable(String) AS Nullable(Numbers)) /// In case when converting to Nullable type, we apply different parsing rule, /// that will not throw an exception but return NULL in case of malformed input. - result_column = ConvertImpl::execute( - arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute( + arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertReturnNullOnErrorTag, scale); return true; } } - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag, scale); return true; }); From 389566f895979104f4931ddb5a2d36be43e1eecb Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Thu, 21 Mar 2024 20:23:06 +0000 Subject: [PATCH 0382/1165] quick fix for style check --- tests/ci/style_check.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index d7f6fa998e9..05788aad5ea 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -8,9 +8,8 @@ import subprocess import sys from concurrent.futures import ProcessPoolExecutor from pathlib import Path -from typing import List, Tuple, Union +from typing import List, Tuple -import magic from docker_images_helper import get_docker_image, pull_image from env_helper import CI, REPO_COPY, TEMP_PATH from git_helper import GIT_PREFIX, git_runner @@ -96,32 +95,34 @@ def commit_push_staged(pr_info: PRInfo) -> None: git_runner(push_cmd) -def is_python(file: Union[Path, str]) -> bool: +def is_python(file: str) -> bool: """returns if the changed file in the repository is python script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - try: - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-script.python" - ) - except IsADirectoryError: - # Process submodules w/o errors - return False + # try: + # return bool( + # magic.from_file(os.path.join(REPO_COPY, file), mime=True) + # == "text/x-script.python" + # ) + # except IsADirectoryError: + # # Process submodules w/o errors + # return False + return file.endswith(".py") -def is_shell(file: Union[Path, str]) -> bool: +def is_shell(file: str) -> bool: """returns if the changed file in the repository is shell script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - try: - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-shellscript" - ) - except IsADirectoryError: - # Process submodules w/o errors - return False + # try: + # return bool( + # magic.from_file(os.path.join(REPO_COPY, file), mime=True) + # == "text/x-shellscript" + # ) + # except IsADirectoryError: + # # Process submodules w/o errors + # return False + return file.endswith(".sh") def main(): From a2836b0887e03bc8c16098ca084a64b145aa634d Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 21 Mar 2024 22:17:18 +0100 Subject: [PATCH 0383/1165] Removed unwanted tests --- ...xx_test_dictionary_create_access.reference | 0 .../xxx_test_dictionary_create_access.sh | 47 ------------------- 2 files changed, 47 deletions(-) delete mode 100644 tests/queries/0_stateless/xxx_test_dictionary_create_access.reference delete mode 100755 tests/queries/0_stateless/xxx_test_dictionary_create_access.sh diff --git a/tests/queries/0_stateless/xxx_test_dictionary_create_access.reference b/tests/queries/0_stateless/xxx_test_dictionary_create_access.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/xxx_test_dictionary_create_access.sh b/tests/queries/0_stateless/xxx_test_dictionary_create_access.sh deleted file mode 100755 index 4478eb59c95..00000000000 --- a/tests/queries/0_stateless/xxx_test_dictionary_create_access.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -readonly PID=$$ -readonly TEST_USER=$"02834_USER_${PID}" - -#${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \ -# -d "SELECT * FROM remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', 'system', 'one', '${TEST_USER}', 'pass')" - -${CLICKHOUSE_CLIENT} -q "CREATE DATABASE db;" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db.table;" - -${CLICKHOUSE_CLIENT} -q "DROP DICTIONARY IF EXISTS db.dict_name;" - -${CLICKHOUSE_CLIENT} -q "CREATE TABLE db.table (label_data_combination_id UInt64) engine = MergeTree Order BY label_data_combination_id;" -${CLICKHOUSE_CLIENT} -q "CREATE USER ${TEST_USER};" -${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON db.* TO ${TEST_USER};" -#${CLICKHOUSE_CLIENT} -q "GRANT INSERT ON db.* TO ${TEST_USER};" -${CLICKHOUSE_CLIENT} -q "GRANT CREATE ON db.* TO ${TEST_USER};" -#${CLICKHOUSE_CLIENT} -q "GRANT DROP ON db.* TO ${TEST_USER};" - -${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \ - -d "CREATE OR REPLACE DICTIONARY db.dict_name - ( - label_data_combination_id UInt64 - ) - PRIMARY KEY label_data_combination_id - SOURCE(CLICKHOUSE( - QUERY - 'SELECT - label_data_combination_id - FROM db.table' - ) - ) - LAYOUT(HASHED(SHARDS 16 SHARD_LOAD_QUEUE_BACKLOG 10000)) - LIFETIME(0) - SETTINGS(dictionary_use_async_executor = 1, max_threads = 16) - COMMENT 'Dictionary mapping of label_data_combination_id to the underlying data.';" - - -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db.table;" -${CLICKHOUSE_CLIENT} -q "DROP DICTIONARY IF EXISTS db.dict_name;" -${CLICKHOUSE_CLIENT} -q "DROP DATABASE db;" From 50bc8acc3724b8fa81d2aa1e7b40515a16268d08 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 21 Mar 2024 21:26:53 +0000 Subject: [PATCH 0384/1165] Automatic style fix --- tests/integration/test_storage_azure_blob_storage/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_azure_blob_storage/test.py b/tests/integration/test_storage_azure_blob_storage/test.py index 6918be9f78a..7d30265e4f8 100644 --- a/tests/integration/test_storage_azure_blob_storage/test.py +++ b/tests/integration/test_storage_azure_blob_storage/test.py @@ -1342,4 +1342,4 @@ def test_parallel_read(cluster): f"select count() from azureBlobStorage('{connection_string}', 'cont', 'test_parallel_read.parquet')", ) assert int(res) == 10000 - assert_logs_contain_with_retry(node, "AzureBlobStorage readBigAt read bytes") \ No newline at end of file + assert_logs_contain_with_retry(node, "AzureBlobStorage readBigAt read bytes") From 257c263596f5832faafff8abfe344c53bf549ea7 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 21 Mar 2024 22:33:34 +0100 Subject: [PATCH 0385/1165] Avoid extra copy of data --- src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 0c4acebf347..2b8986a265e 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -259,7 +259,7 @@ size_t ReadBufferFromAzureBlobStorage::getFileSize() return *file_size; } -size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t range_begin, const std::function & progress_callback) const +size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t range_begin, const std::function & /*progress_callback*/) const { size_t initial_n = n; @@ -275,9 +275,7 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran auto download_response = blob_client->Download(download_options); std::unique_ptr body_stream = std::move(download_response.Value.BodyStream); - auto bytes = body_stream->ReadToCount(reinterpret_cast(data_ptr), body_stream->Length()); - std::istringstream string_stream(String(static_cast(data_ptr),bytes)); // STYLE_CHECK_ALLOW_STD_STRING_STREAM - copyFromIStreamWithProgressCallback(string_stream, to, n, progress_callback, &bytes_copied); + bytes_copied = body_stream->ReadToCount(reinterpret_cast(to), body_stream->Length()); LOG_INFO(log, "AzureBlobStorage readBigAt read bytes {}", bytes_copied); From 7ac41d7fc195004b4efccbd7891466fe956500e1 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Thu, 21 Mar 2024 23:25:31 +0000 Subject: [PATCH 0386/1165] Fix segfault in SquashingTransform --- src/Interpreters/SquashingTransform.cpp | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/SquashingTransform.cpp b/src/Interpreters/SquashingTransform.cpp index 4ed0dddc191..da608f5e7ce 100644 --- a/src/Interpreters/SquashingTransform.cpp +++ b/src/Interpreters/SquashingTransform.cpp @@ -89,13 +89,25 @@ void SquashingTransform::append(ReferenceType input_block) assert(blocksHaveEqualStructure(input_block, accumulated_block)); - for (size_t i = 0, size = accumulated_block.columns(); i < size; ++i) + try { - const auto source_column = input_block.getByPosition(i).column; + for (size_t i = 0, size = accumulated_block.columns(); i < size; ++i) + { + const auto source_column = input_block.getByPosition(i).column; - auto mutable_column = IColumn::mutate(std::move(accumulated_block.getByPosition(i).column)); - mutable_column->insertRangeFrom(*source_column, 0, source_column->size()); - accumulated_block.getByPosition(i).column = std::move(mutable_column); + auto mutable_column = IColumn::mutate(std::move(accumulated_block.getByPosition(i).column)); + mutable_column->insertRangeFrom(*source_column, 0, source_column->size()); + accumulated_block.getByPosition(i).column = std::move(mutable_column); + } + } + catch (...) + { + /// add() may be called again even after a previous add() threw an exception. + /// Keep accumulated_block in a valid state. + /// Seems ok to discard accumulated data because we're throwing an exception, which the caller will + /// hopefully interpret to mean "this block and all *previous* blocks are potentially lost". + accumulated_block.clear(); + throw; } } @@ -107,6 +119,9 @@ bool SquashingTransform::isEnoughSize(const Block & block) for (const auto & [column, type, name] : block) { + if (!column) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid column in block."); + if (!rows) rows = column->size(); else if (rows != column->size()) From 415ed79b1673c06ac6d50114ce33ca6e12c198ec Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Fri, 15 Dec 2023 01:46:08 +0000 Subject: [PATCH 0387/1165] allow declaring enum in external table structure Signed-off-by: Duc Canh Le --- src/Core/ExternalTable.cpp | 73 ++++++++++++++++--- .../02935_external_table_enum_type.reference | 2 + .../02935_external_table_enum_type.sh | 10 +++ 3 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/02935_external_table_enum_type.reference create mode 100755 tests/queries/0_stateless/02935_external_table_enum_type.sh diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index 58b705ca317..fa1e49d437c 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -17,6 +17,9 @@ #include #include +#include +#include +#include #include @@ -28,6 +31,18 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +/// Parsing a list of types with `,` as separator. For example, `Int, Enum('foo'=1,'bar'=2), Double` +/// Used in `parseStructureFromTypesField` +class ParserTypeList : public IParserBase +{ +protected: + const char * getName() const override { return "type pair list"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override + { + return ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) + .parse(pos, node, expected); + } +}; ExternalTableDataPtr BaseExternalTable::getData(ContextPtr context) { @@ -55,23 +70,61 @@ void BaseExternalTable::clear() void BaseExternalTable::parseStructureFromStructureField(const std::string & argument) { - std::vector vals; - splitInto<' ', ','>(vals, argument, true); + /// First try to parse table structure with `ParserNameTypePairList`, this allows user to declare Enum types in the structure + ParserNameTypePairList parser; + const auto * pos = argument.data(); + String error; + ASTPtr columns_list_raw = tryParseQuery(parser, pos, pos+argument.size(), error, false, "", false, 0, 0); + bool parse_structure_with_parser = false; + if ((parse_structure_with_parser = columns_list_raw != nullptr)) + { + for (auto & child : columns_list_raw->children) + { + auto * column = child->as(); + if (column) + structure.emplace_back(column->name, column->type->getColumnNameWithoutAlias()); + else + { + structure.clear(); + parse_structure_with_parser = false; + break; + } + } + } - if (vals.size() % 2 != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Odd number of attributes in section structure: {}", vals.size()); + if (!parse_structure_with_parser) + { + std::vector vals; + splitInto<' ', ','>(vals, argument, true); - for (size_t i = 0; i < vals.size(); i += 2) - structure.emplace_back(vals[i], vals[i + 1]); + if (vals.size() % 2 != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Odd number of attributes in section structure: {}", vals.size()); + + for (size_t i = 0; i < vals.size(); i += 2) + structure.emplace_back(vals[i], vals[i + 1]); + } } void BaseExternalTable::parseStructureFromTypesField(const std::string & argument) { - std::vector vals; - splitInto<' ', ','>(vals, argument, true); + /// First try to parse table structure with `ParserTypeList`, this allows user to declare Enum types in the structure + ParserTypeList parser; + const auto * pos = argument.data(); + String error; + ASTPtr type_list_raw = tryParseQuery(parser, pos, pos+argument.size(), error, false, "", false, 0, 0); + if (type_list_raw != nullptr) + { + for (size_t i = 0; i < type_list_raw->children.size(); ++i) + structure.emplace_back("_" + toString(i + 1), type_list_raw->children[i]->getColumnNameWithoutAlias()); + } + else + { + std::vector vals; + splitInto<' ', ','>(vals, argument, true); - for (size_t i = 0; i < vals.size(); ++i) - structure.emplace_back("_" + toString(i + 1), vals[i]); + for (size_t i = 0; i < vals.size(); ++i) + structure.emplace_back("_" + toString(i + 1), vals[i]); + } } void BaseExternalTable::initSampleBlock() diff --git a/tests/queries/0_stateless/02935_external_table_enum_type.reference b/tests/queries/0_stateless/02935_external_table_enum_type.reference new file mode 100644 index 00000000000..1efccdbfc67 --- /dev/null +++ b/tests/queries/0_stateless/02935_external_table_enum_type.reference @@ -0,0 +1,2 @@ +foo 1 +bar 2 diff --git a/tests/queries/0_stateless/02935_external_table_enum_type.sh b/tests/queries/0_stateless/02935_external_table_enum_type.sh new file mode 100755 index 00000000000..ab4306a056e --- /dev/null +++ b/tests/queries/0_stateless/02935_external_table_enum_type.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +http_url="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?" + +curl -s "${http_url}temp_structure=x+Enum8('foo'%3D1,'bar'%3D2),y+Int" -F "$(printf 'temp='foo'\t1');filename=data1" -F "query=SELECT * FROM temp" +curl -s "${http_url}temp_types=Enum8('foo'%3D1,'bar'%3D2),Int" -F "$(printf 'temp='bar'\t2');filename=data1" -F "query=SELECT * FROM temp" From 3f0bba97a68016803c9c730a47872468a7c3d997 Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Wed, 20 Dec 2023 03:14:47 +0000 Subject: [PATCH 0388/1165] fix shellcheck Signed-off-by: Duc Canh Le --- tests/queries/0_stateless/02935_external_table_enum_type.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02935_external_table_enum_type.sh b/tests/queries/0_stateless/02935_external_table_enum_type.sh index ab4306a056e..292fb647b27 100755 --- a/tests/queries/0_stateless/02935_external_table_enum_type.sh +++ b/tests/queries/0_stateless/02935_external_table_enum_type.sh @@ -6,5 +6,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) http_url="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?" -curl -s "${http_url}temp_structure=x+Enum8('foo'%3D1,'bar'%3D2),y+Int" -F "$(printf 'temp='foo'\t1');filename=data1" -F "query=SELECT * FROM temp" -curl -s "${http_url}temp_types=Enum8('foo'%3D1,'bar'%3D2),Int" -F "$(printf 'temp='bar'\t2');filename=data1" -F "query=SELECT * FROM temp" +curl -s "${http_url}temp_structure=x+Enum8('foo'%3D1,'bar'%3D2),y+Int" -F "$(printf 'temp='"foo"'\t1');filename=data1" -F "query=SELECT * FROM temp" +curl -s "${http_url}temp_types=Enum8('foo'%3D1,'bar'%3D2),Int" -F "$(printf 'temp='"bar"'\t2');filename=data1" -F "query=SELECT * FROM temp" From f45e3ba4327c09fd67746a15bdfa82ad79390dad Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Mon, 4 Mar 2024 15:37:41 +0000 Subject: [PATCH 0389/1165] address review comments Signed-off-by: Duc Canh Le --- src/Core/ExternalTable.cpp | 57 ++++++------------- .../02935_external_table_enum_type.reference | 2 + .../02935_external_table_enum_type.sh | 2 + 3 files changed, 20 insertions(+), 41 deletions(-) diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index fa1e49d437c..23a181d8ea2 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -70,61 +70,36 @@ void BaseExternalTable::clear() void BaseExternalTable::parseStructureFromStructureField(const std::string & argument) { - /// First try to parse table structure with `ParserNameTypePairList`, this allows user to declare Enum types in the structure ParserNameTypePairList parser; const auto * pos = argument.data(); String error; - ASTPtr columns_list_raw = tryParseQuery(parser, pos, pos+argument.size(), error, false, "", false, 0, 0); - bool parse_structure_with_parser = false; - if ((parse_structure_with_parser = columns_list_raw != nullptr)) + ASTPtr columns_list_raw = tryParseQuery(parser, pos, pos + argument.size(), error, false, "", false, 0, 0); + + if (!columns_list_raw) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: {}", error); + + for (auto & child : columns_list_raw->children) { - for (auto & child : columns_list_raw->children) - { - auto * column = child->as(); - if (column) - structure.emplace_back(column->name, column->type->getColumnNameWithoutAlias()); - else - { - structure.clear(); - parse_structure_with_parser = false; - break; - } - } - } - - if (!parse_structure_with_parser) - { - std::vector vals; - splitInto<' ', ','>(vals, argument, true); - - if (vals.size() % 2 != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Odd number of attributes in section structure: {}", vals.size()); - - for (size_t i = 0; i < vals.size(); i += 2) - structure.emplace_back(vals[i], vals[i + 1]); + auto * column = child->as(); + if (column) + structure.emplace_back(column->name, column->type->getColumnNameWithoutAlias()); + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: expected column definition, got {}", child->formatForErrorMessage()); } } void BaseExternalTable::parseStructureFromTypesField(const std::string & argument) { - /// First try to parse table structure with `ParserTypeList`, this allows user to declare Enum types in the structure ParserTypeList parser; const auto * pos = argument.data(); String error; ASTPtr type_list_raw = tryParseQuery(parser, pos, pos+argument.size(), error, false, "", false, 0, 0); - if (type_list_raw != nullptr) - { - for (size_t i = 0; i < type_list_raw->children.size(); ++i) - structure.emplace_back("_" + toString(i + 1), type_list_raw->children[i]->getColumnNameWithoutAlias()); - } - else - { - std::vector vals; - splitInto<' ', ','>(vals, argument, true); - for (size_t i = 0; i < vals.size(); ++i) - structure.emplace_back("_" + toString(i + 1), vals[i]); - } + if (!type_list_raw) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: {}", error); + + for (size_t i = 0; i < type_list_raw->children.size(); ++i) + structure.emplace_back("_" + toString(i + 1), type_list_raw->children[i]->getColumnNameWithoutAlias()); } void BaseExternalTable::initSampleBlock() diff --git a/tests/queries/0_stateless/02935_external_table_enum_type.reference b/tests/queries/0_stateless/02935_external_table_enum_type.reference index 1efccdbfc67..ed152e608b9 100644 --- a/tests/queries/0_stateless/02935_external_table_enum_type.reference +++ b/tests/queries/0_stateless/02935_external_table_enum_type.reference @@ -1,2 +1,4 @@ foo 1 bar 2 +foo 1 +bar 2 diff --git a/tests/queries/0_stateless/02935_external_table_enum_type.sh b/tests/queries/0_stateless/02935_external_table_enum_type.sh index 292fb647b27..61d5508e9f9 100755 --- a/tests/queries/0_stateless/02935_external_table_enum_type.sh +++ b/tests/queries/0_stateless/02935_external_table_enum_type.sh @@ -8,3 +8,5 @@ http_url="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?" curl -s "${http_url}temp_structure=x+Enum8('foo'%3D1,'bar'%3D2),y+Int" -F "$(printf 'temp='"foo"'\t1');filename=data1" -F "query=SELECT * FROM temp" curl -s "${http_url}temp_types=Enum8('foo'%3D1,'bar'%3D2),Int" -F "$(printf 'temp='"bar"'\t2');filename=data1" -F "query=SELECT * FROM temp" +echo -ne 'foo\t1' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Enum8('foo'=1,'bar'=2),y Int" +echo -ne 'bar\t2' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --types="Enum8('foo'=1,'bar'=2),Int" From e39576917dfe2ba69c5181035c87182d2404aee5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 22 Mar 2024 00:40:28 +0100 Subject: [PATCH 0390/1165] Add a test --- .../03018_external_with_complex_data_types.reference | 1 + .../0_stateless/03018_external_with_complex_data_types.sh | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/03018_external_with_complex_data_types.reference create mode 100755 tests/queries/0_stateless/03018_external_with_complex_data_types.sh diff --git a/tests/queries/0_stateless/03018_external_with_complex_data_types.reference b/tests/queries/0_stateless/03018_external_with_complex_data_types.reference new file mode 100644 index 00000000000..11277a62b06 --- /dev/null +++ b/tests/queries/0_stateless/03018_external_with_complex_data_types.reference @@ -0,0 +1 @@ +Hello world diff --git a/tests/queries/0_stateless/03018_external_with_complex_data_types.sh b/tests/queries/0_stateless/03018_external_with_complex_data_types.sh new file mode 100755 index 00000000000..fb3a22d9044 --- /dev/null +++ b/tests/queries/0_stateless/03018_external_with_complex_data_types.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --external --file <(echo "Hello, world") --name test --format CSV --structure "x Enum('Hello' = 1, 'world' = 2), y String" --query "SELECT * FROM test" From 828c555780c70a260029b13d07e61f2cdbf493a6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 22 Mar 2024 00:42:49 +0100 Subject: [PATCH 0391/1165] Fix build --- src/Core/ExternalTable.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index 23a181d8ea2..e043a2f9492 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -73,7 +73,7 @@ void BaseExternalTable::parseStructureFromStructureField(const std::string & arg ParserNameTypePairList parser; const auto * pos = argument.data(); String error; - ASTPtr columns_list_raw = tryParseQuery(parser, pos, pos + argument.size(), error, false, "", false, 0, 0); + ASTPtr columns_list_raw = tryParseQuery(parser, pos, pos + argument.size(), error, false, "", false, DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, true); if (!columns_list_raw) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: {}", error); @@ -93,7 +93,7 @@ void BaseExternalTable::parseStructureFromTypesField(const std::string & argumen ParserTypeList parser; const auto * pos = argument.data(); String error; - ASTPtr type_list_raw = tryParseQuery(parser, pos, pos+argument.size(), error, false, "", false, 0, 0); + ASTPtr type_list_raw = tryParseQuery(parser, pos, pos+argument.size(), error, false, "", false, DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, true); if (!type_list_raw) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: {}", error); From b91d478c937fe73a7f92585c3b8d30b8732d5975 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 22 Mar 2024 00:56:57 +0100 Subject: [PATCH 0392/1165] There is no such thing as broken tests --- tests/analyzer_integration_broken_tests.txt | 0 tests/integration/ci-runner.py | 32 ++++----------------- 2 files changed, 6 insertions(+), 26 deletions(-) delete mode 100644 tests/analyzer_integration_broken_tests.txt diff --git a/tests/analyzer_integration_broken_tests.txt b/tests/analyzer_integration_broken_tests.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index e7f691d2237..32a60c8fa47 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -475,7 +475,7 @@ class ClickhouseIntegrationTestsRunner: return result @staticmethod - def _update_counters(main_counters, current_counters, broken_tests): + def _update_counters(main_counters, current_counters): for test in current_counters["PASSED"]: if test not in main_counters["PASSED"]: if test in main_counters["FAILED"]: @@ -483,21 +483,14 @@ class ClickhouseIntegrationTestsRunner: if test in main_counters["BROKEN"]: main_counters["BROKEN"].remove(test) - if test not in broken_tests: - main_counters["PASSED"].append(test) - else: - main_counters["NOT_FAILED"].append(test) + main_counters["PASSED"].append(test) for state in ("ERROR", "FAILED"): for test in current_counters[state]: if test in main_counters["PASSED"]: main_counters["PASSED"].remove(test) - if test not in broken_tests: - if test not in main_counters[state]: - main_counters[state].append(test) - else: - if test not in main_counters["BROKEN"]: - main_counters["BROKEN"].append(test) + if test not in main_counters[state]: + main_counters[state].append(test) for state in ("SKIPPED",): for test in current_counters[state]: @@ -564,7 +557,6 @@ class ClickhouseIntegrationTestsRunner: tests_in_group, num_tries, num_workers, - broken_tests, ): try: return self.run_test_group( @@ -573,7 +565,6 @@ class ClickhouseIntegrationTestsRunner: tests_in_group, num_tries, num_workers, - broken_tests, ) except Exception as e: logging.info("Failed to run %s:\n%s", test_group, e) @@ -596,7 +587,6 @@ class ClickhouseIntegrationTestsRunner: tests_in_group, num_tries, num_workers, - broken_tests, ): counters = { "ERROR": [], @@ -706,7 +696,7 @@ class ClickhouseIntegrationTestsRunner: ) times_lines = parse_test_times(info_path) new_tests_times = get_test_times(times_lines) - self._update_counters(counters, new_counters, broken_tests) + self._update_counters(counters, new_counters) for test_name, test_time in new_tests_times.items(): tests_times[test_name] = test_time @@ -915,20 +905,10 @@ class ClickhouseIntegrationTestsRunner: logging.info("Shuffling test groups") random.shuffle(items_to_run) - broken_tests = [] - if self.use_analyzer: - with open( - f"{repo_path}/tests/analyzer_integration_broken_tests.txt", - "r", - encoding="utf-8", - ) as f: - broken_tests = f.read().splitlines() - logging.info("Broken tests in the list: %s", len(broken_tests)) - for group, tests in items_to_run: logging.info("Running test group %s containing %s tests", group, len(tests)) group_counters, group_test_times, log_paths = self.try_run_test_group( - repo_path, group, tests, MAX_RETRY, NUM_WORKERS, broken_tests + repo_path, group, tests, MAX_RETRY, NUM_WORKERS ) total_tests = 0 for counter, value in group_counters.items(): From 7d7025a74edf9c779cad98ca87d32252c5213e31 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Thu, 21 Mar 2024 22:20:33 +0000 Subject: [PATCH 0393/1165] CI: modify it --- .github/PULL_REQUEST_TEMPLATE.md | 53 ++++++++++++++++---------------- tests/ci/ci.py | 9 +++--- tests/ci/ci_config.py | 46 ++++++++++++++++++++++++++- 3 files changed, 75 insertions(+), 33 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4db0737c959..fede3fe519d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -43,30 +43,29 @@ At a minimum, the following information should be added (but add more as needed) --- ### Modify your CI run: -##### NOTE: -- if your merge the PR with modified CI you **MUST** know what you are doing. -- modifiers can be applied only if set before CI starts -- remove `!` to apply -- return all `!` to restore defaults -``` -!#ci_set_ - to run only preconfigured set of tests, e.g.: -!#ci_set_arm - to run only integration tests on ARM -!#ci_set_integration - to run only integration tests on AMD -!#ci_set_analyzer - to run only tests for analyzer -NOTE: you can configure your own ci set -``` -``` -!#job_ - to run only specified job, e.g.: -!#job_stateless_tests_release -!#job_package_debug -!#job_style_check -!#job_integration_tests_asan -``` -``` -!#batch_2 - to run only 2nd batch for all multi-batch jobs -!#btach_1_2_3 - to run only 1, 2, 3rd batch for all multi-batch jobs -``` -``` -!#no_merge_commit - to disable merge commit (no merge from master) -!#do_not_test - to disable whole CI (except style check) -``` +**NOTE:** If your merge the PR with modified CI you **MUST KNOW** what you are doing +**NOTE:** Set desired options before CI starts or re-push after updates + +#### Run only: +- [ ] Integration tests +- [ ] Integration tests (arm64) +- [ ] Stateless tests (release) +- [ ] Stateless tests (asan) +- [ ] Stateful tests (release) +- [ ] Stateful tests (asan) +- [ ] No sanitizers +- [ ] Tests with analyzer +- [ ] Fast tests +- [ ] Only package_debug build +- [ ] Add your CI variant description here + +#### CI options: +- [ ] do not test (only style check) +- [ ] disable merge-commit (no merge from master before tests) +- [ ] disable CI cache (job reuse) + +#### Only specified batches in multi-batch jobs: +- [ ] 1 +- [ ] 2 +- [ ] 3 +- [ ] 4 diff --git a/tests/ci/ci.py b/tests/ci/ci.py index cd63514cb6a..514189a8b8a 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1248,9 +1248,7 @@ def _configure_jobs( for token_ in ci_controlling_tokens: label_config = CI_CONFIG.get_label_config(token_) assert label_config, f"Unknonwn token [{token_}]" - print( - f"NOTE: CI controlling token: [{ci_controlling_tokens}], add jobs: [{label_config.run_jobs}]" - ) + print(f"NOTE: CI modifier: [{token_}], add jobs: [{label_config.run_jobs}]") jobs_to_do_requested += label_config.run_jobs # handle specific job requests @@ -1264,7 +1262,7 @@ def _configure_jobs( for job in requested_jobs: job_with_parents = CI_CONFIG.get_job_with_parents(job) print( - f"NOTE: CI controlling token: [#job_{job}], add jobs: [{job_with_parents}]" + f"NOTE: CI modifier: [#job_{job}], add jobs: [{job_with_parents}]" ) # always add requested job itself, even if it could be skipped jobs_to_do_requested.append(job_with_parents[0]) @@ -1273,6 +1271,7 @@ def _configure_jobs( jobs_to_do_requested.append(parent) if jobs_to_do_requested: + jobs_to_do_requested = list(set(jobs_to_do_requested)) print( f"NOTE: Only specific job(s) were requested by commit message tokens: [{jobs_to_do_requested}]" ) @@ -1408,7 +1407,7 @@ def _update_gh_statuses_action(indata: Dict, s3: S3Helper) -> None: def _fetch_commit_tokens(message: str, pr_info: PRInfo) -> List[str]: - pattern = r"([^!]|^)#(\w+)" + pattern = r"(#|- \[x\] + Integration tests diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 514189a8b8a..bd1582261a8 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -12,7 +12,7 @@ from copy import deepcopy from dataclasses import asdict, dataclass from enum import Enum from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Set, Union +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union import docker_images_helper import upload_result_helper @@ -733,6 +733,183 @@ class CiCache: return await_finished +@dataclass +class CiOptions: + # job will be included in the run if any keyword from the list matches job name + include_keywords: Optional[List[str]] = None + # job will be excluded in the run if any keyword from the list matches job name + exclude_keywords: Optional[List[str]] = None + + # list of specified preconfigured ci sets to run + ci_sets: Optional[List[str]] = None + # list of specified jobs to run + ci_jobs: Optional[List[str]] = None + + # btaches to run for all multi-batch jobs + job_batches: Optional[List[int]] = None + + do_not_test: bool = False + no_ci_cache: bool = False + no_merge_commit: bool = False + + def as_dict(self) -> Dict[str, Any]: + return asdict(self) + + @staticmethod + def create_from_run_config(run_config: Dict[str, Any]) -> "CiOptions": + return CiOptions(**run_config["ci_options"]) + + @staticmethod + def create_from_pr_message(commit_message: Optional[str]) -> "CiOptions": + """ + Creates CiOptions instance based on tags found in PR body and/or commit message + @commit_message - may be provided directly for debugging purposes, otherwise it will be retrieved from git. + """ + res = CiOptions() + pr_info = PRInfo() + if not pr_info.is_pr(): + # CI options can be configured in PRs only + return res + + message = commit_message or GitRunner(set_cwd_to_git_root=True).run( + f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1" + ) + + pattern = r"(#|- \[x\] + Integration tests -- [ ] Integration tests (arm64) -- [ ] Stateless tests (release) -- [ ] Stateless tests (asan) -- [ ] Stateful tests (release) -- [ ] Stateful tests (asan) -- [ ] No sanitizers -- [ ] Tests with analyzer -- [ ] Fast tests -- [ ] Only package_debug build -- [ ] Add your CI variant description here +#### Include tests (required builds will be added automatically): +- [ ] Fast test +- [ ] Integration Tests +- [ ] Stateless tests +- [ ] Stateful tests +- [ ] Unit tests +- [ ] Performance tests +- [ ] All with ASAN +- [ ] All with TSAN +- [ ] All with Analyzer +- [ ] Add your option here -#### CI options: +#### Exclude tests: +- [ ] Fast test +- [ ] Integration Tests +- [ ] Stateless tests +- [ ] Stateful tests +- [ ] Performance tests +- [ ] All with ASAN +- [ ] All with TSAN +- [ ] All with MSAN +- [ ] All with UBSAN +- [ ] All with Coverage +- [ ] All with Aarch64 +- [ ] Add your option here + +#### Extra options: - [ ] do not test (only style check) - [ ] disable merge-commit (no merge from master before tests) - [ ] disable CI cache (job reuse) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index bd1582261a8..a3214784d29 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -760,18 +760,21 @@ class CiOptions: return CiOptions(**run_config["ci_options"]) @staticmethod - def create_from_pr_message(commit_message: Optional[str]) -> "CiOptions": + def create_from_pr_message( + debug_message: Optional[str], update_from_api: bool + ) -> "CiOptions": """ Creates CiOptions instance based on tags found in PR body and/or commit message @commit_message - may be provided directly for debugging purposes, otherwise it will be retrieved from git. """ res = CiOptions() pr_info = PRInfo() - if not pr_info.is_pr(): + if ( + not pr_info.is_pr() and not debug_message + ): # if commit_message is provided it's test/debug scenario - do not return # CI options can be configured in PRs only return res - - message = commit_message or GitRunner(set_cwd_to_git_root=True).run( + message = debug_message or GitRunner(set_cwd_to_git_root=True).run( f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1" ) @@ -779,10 +782,13 @@ class CiOptions: matches = [match[-1] for match in re.findall(pattern, message)] print(f"CI tags from commit message: [{matches}]") - pr_info = PRInfo(pr_event_from_api=True) # Fetch updated PR body from GH API - matches_pr = [match[-1] for match in re.findall(pattern, pr_info.body)] - print(f"CI tags from PR body: [{matches_pr}]") - matches = set(matches + matches_pr) + if not debug_message: # to be skipped if debug/test + pr_info = PRInfo( + pr_event_from_api=update_from_api + ) # Fetch updated PR body from GH API + matches_pr = [match[-1] for match in re.findall(pattern, pr_info.body)] + print(f"CI tags from PR body: [{matches_pr}]") + matches = list(set(matches + matches_pr)) if "do not test" in pr_info.labels: # do_not_test could be set in GH labels @@ -800,11 +806,15 @@ class CiOptions: elif match.startswith("ci_include_"): if not res.include_keywords: res.include_keywords = [] - res.include_keywords.append(match.removeprefix("ci_include_")) + res.include_keywords.append( + normalize_check_name(match.removeprefix("ci_include_")) + ) elif match.startswith("ci_exclude_"): if not res.exclude_keywords: res.exclude_keywords = [] - res.exclude_keywords.append(match.removeprefix("ci_exclude_")) + res.exclude_keywords.append( + normalize_check_name(match.removeprefix("ci_exclude_")) + ) elif match == Labels.NO_CI_CACHE: res.no_ci_cache = True print("NOTE: CI Cache will be disabled") @@ -845,6 +855,44 @@ class CiOptions: """ jobs_to_do_requested = [] # type: List[str] + # -1. Handle "ci_exclude_" tags if any + if self.exclude_keywords: + new_jobs_to_do = list(jobs_to_do) + for job in jobs_to_do: + found = False + for keyword in self.exclude_keywords: + if keyword in normalize_check_name(job): + print( + f"Job [{job}] matches Exclude keyword [{keyword}] - remove" + ) + found = True + break + if found: + new_jobs_to_do.remove(job) + jobs_to_do = new_jobs_to_do + + # 0. Handle "ci_include_" tags if any + if self.include_keywords: + for job in jobs_to_do: + found = False + for keyword in self.include_keywords: + if keyword in normalize_check_name(job): + print(f"Job [{job}] matches Include keyword [{keyword}] - add") + found = True + break + if found: + job_with_parents = CI_CONFIG.get_job_with_parents(job) + for job in job_with_parents: + if job in jobs_to_do and job not in jobs_to_do_requested: + jobs_to_do_requested.append(job) + assert ( + jobs_to_do_requested + ), "Include tags are set but now job configured - Invalid tags, probably [{self.include_keywords}]" + if JobNames.STYLE_CHECK not in jobs_to_do_requested: + # Style check must not be omitted + jobs_to_do_requested.append(JobNames.STYLE_CHECK) + + # FIXME: to be removed in favor of include/exclude # 1. Handle "ci_set_" tags if any if self.ci_sets: for tag in self.ci_sets: @@ -855,6 +903,7 @@ class CiOptions: ) jobs_to_do_requested += label_config.run_jobs + # FIXME: to be removed in favor of include/exclude # 2. Handle "job_" tags if any if self.ci_jobs: for tag in self.ci_jobs: @@ -870,6 +919,8 @@ class CiOptions: # 3. Handle "do not test" if self.do_not_test: + label_config = CI_CONFIG.get_label_config(Labels.DO_NOT_TEST_LABEL) + assert label_config print( f"NOTE: CI 'do not test' setting applied, set jobs: [{label_config.run_jobs}]" ) @@ -877,8 +928,7 @@ class CiOptions: print( "WARNING: 'do not test' is used alongside with other CI modifying tags - 'do not test' prevails" ) - label_config = CI_CONFIG.get_label_config(Labels.DO_NOT_TEST_LABEL) - jobs_to_do_requested = label_config.run_jobs + jobs_to_do_requested = list(label_config.run_jobs) if jobs_to_do_requested: jobs_to_do_requested = list(set(jobs_to_do_requested)) @@ -1794,7 +1844,9 @@ def main() -> int: ### CONFIGURE action: start if args.configure: - ci_options = CiOptions.create_from_pr_message(args.commit_message or None) + ci_options = CiOptions.create_from_pr_message( + args.commit_message or None, update_from_api=True + ) # tokens = _fetch_commit_tokens(message, pr_info) if ci_options.no_merge_commit and CI: diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py new file mode 100644 index 00000000000..61aeec59581 --- /dev/null +++ b/tests/ci/test_ci_options.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 + +# type: ignore + +import unittest +from ci import CiOptions +from ci_config import JobNames + +_TEST_BODY_1 = """ +#### Run only: +- [x] Integration tests +- [ ] Integration tests (arm64) +- [x] Integration tests +- [x] Integration tests +- [ ] Integration tests +- [x] some invalid mask - should be skipped +- [x] Integration tests +- [ ] Integration tests + +#### CI options: +- [ ] do not test (only style check) +- [x] disable merge-commit (no merge from master before tests) +- [ ] disable CI cache (job reuse) + +#### Only specified batches in multi-batch jobs: +- [x] 1 +- [ ] 2 +""" + +_TEST_BODY_2 = """ +- [x] MUST include integration tests +- [x] MUST include stateless tests +- [x] no action must be applied +- [ ] no action must be applied +- [x] MUST exclude tsan +- [x] MUST exclude aarch64 +- [x] MUST exclude test with analazer +- [ ] no action applied +- [x] Must exclude statless test with s3 storage +- [x] Must exclude tests on coverage build +""" + +_TEST_BODY_3 = """ +- [x] Must include all tests for analyzer +""" + + +class TestCIOptions(unittest.TestCase): + def test_pr_body_parsing(self): + ci_options = CiOptions.create_from_pr_message( + _TEST_BODY_1, update_from_api=False + ) + self.assertFalse(ci_options.do_not_test) + self.assertFalse(ci_options.no_ci_cache) + self.assertTrue(ci_options.no_merge_commit) + self.assertEqual(ci_options.ci_sets, ["ci_set_integration"]) + self.assertCountEqual(ci_options.include_keywords, ["foo", "foo_bar"]) + self.assertCountEqual(ci_options.exclude_keywords, ["foo", "foo_bar"]) + + def test_options_applied(self): + self.maxDiff = None + ci_options = CiOptions.create_from_pr_message( + _TEST_BODY_2, update_from_api=False + ) + self.assertCountEqual( + ci_options.include_keywords, ["integration", "foo_bar", "stateless"] + ) + self.assertCountEqual( + ci_options.exclude_keywords, + ["tsan", "aarch64", "analyzer", "s3_storage", "coverage"], + ) + jobs_to_do = list(JobNames) + jobs_to_skip = [] + job_params = {} + jobs_to_do, jobs_to_skip, job_params = ci_options.apply( + jobs_to_do, jobs_to_skip, job_params + ) + self.assertCountEqual( + jobs_to_do, + [ + "Style check", + "package_release", + "package_asan", + "package_ubsan", + "package_debug", + "package_msan", + "Stateless tests (asan)", + "Stateless tests flaky check (asan)", + "Stateless tests (msan)", + "Stateless tests (ubsan)", + "Stateless tests (debug)", + "Stateless tests (release)", + "Integration tests (release)", + "Integration tests (asan)", + "Integration tests flaky check (asan)", + ], + ) + + def test_options_applied_2(self): + self.maxDiff = None + ci_options = CiOptions.create_from_pr_message( + _TEST_BODY_3, update_from_api=False + ) + self.assertCountEqual(ci_options.include_keywords, ["analyzer"]) + self.assertIsNone(ci_options.exclude_keywords) + jobs_to_do = list(JobNames) + jobs_to_skip = [] + job_params = {} + jobs_to_do, jobs_to_skip, job_params = ci_options.apply( + jobs_to_do, jobs_to_skip, job_params + ) + self.assertCountEqual( + jobs_to_do, + [ + "Style check", + "Integration tests (asan, analyzer)", + "package_release", + "Stateless tests (release, analyzer, s3, DatabaseReplicated)", + "package_asan", + ], + ) From 3bc91476290e46f140daff1af40043b794fef88a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 26 Mar 2024 20:56:39 +0100 Subject: [PATCH 0680/1165] Update 02916_move_partition_inactive_replica.sql --- .../0_stateless/02916_move_partition_inactive_replica.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql index ca153eea221..b0699539ac1 100644 --- a/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql +++ b/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql @@ -8,8 +8,8 @@ drop table if exists shard_1.from_1; drop table if exists shard_0.to; drop table if exists shard_1.to; -create table shard_0.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; -create table shard_1.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; +create table shard_0.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, shared_merge_tree_disable_merges_and_mutations_assignment=1; +create table shard_1.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, shared_merge_tree_disable_merges_and_mutations_assignment=1; system stop merges shard_0.from_1; system stop merges shard_1.from_1; From 2ce0d647f28a9b2950cb8cf9f5d5ed42f4eb6f68 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 26 Mar 2024 21:39:23 +0100 Subject: [PATCH 0681/1165] Small test fix --- tests/integration/test_broken_projections/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py index b4e8b6e5021..5a6232819b6 100644 --- a/tests/integration/test_broken_projections/test.py +++ b/tests/integration/test_broken_projections/test.py @@ -366,7 +366,7 @@ def test_broken_ignored(cluster): ) assert "all_3_3_0" in get_broken_projections_info(node, table_name, active=False) - assert "all_2_2_0" in get_broken_projections_info(node, table_name, active=False) + assert "all_2_2_0" in get_broken_projections_info(node, table_name, active=True) # 0 because of all_2_2_0 check(node, table_name, 0) From dc83eb1ad66649d35e4f3ddc8525f7896032c3d2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 26 Mar 2024 21:41:32 +0100 Subject: [PATCH 0682/1165] Review fixes --- src/Interpreters/Cache/EvictionCandidates.cpp | 4 ++-- src/Interpreters/Cache/IFileCachePriority.cpp | 10 -------- src/Interpreters/Cache/IFileCachePriority.h | 23 ++++++++++--------- src/Interpreters/Cache/Metadata.h | 12 ++++++++-- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 037b6985be5..cb15af72704 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -39,7 +39,7 @@ EvictionCandidates::~EvictionCandidates() // Reset the evicting state // (as the corresponding file segments were not yet removed). for (const auto & candidate : key_candidates.candidates) - candidate->setEvicting(false, nullptr, nullptr); + candidate->resetEvictingFlag(); } } @@ -53,7 +53,7 @@ void EvictionCandidates::add( it->second.key_metadata = locked_key.getKeyMetadata(); it->second.candidates.push_back(candidate); - candidate->setEvicting(true, &locked_key, &lock); + candidate->setEvictingFlag(locked_key, lock); ++candidates_size; } diff --git a/src/Interpreters/Cache/IFileCachePriority.cpp b/src/Interpreters/Cache/IFileCachePriority.cpp index 8cd2c49c178..6532af55ed2 100644 --- a/src/Interpreters/Cache/IFileCachePriority.cpp +++ b/src/Interpreters/Cache/IFileCachePriority.cpp @@ -13,7 +13,6 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; - extern const int NOT_IMPLEMENTED; } IFileCachePriority::IFileCachePriority(size_t max_size_, size_t max_elements_) @@ -52,13 +51,4 @@ void IFileCachePriority::check(const CachePriorityGuard::Lock & lock) const } } -void IFileCachePriority::holdImpl(size_t, size_t, const CachePriorityGuard::Lock &) -{ - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method holdImpl() is not implemented"); -} - -void IFileCachePriority::releaseImpl(size_t, size_t) -{ - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method holdImpl() is not implemented"); -} } diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index c12f79e3d7a..ddb1cf3c409 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -52,14 +52,18 @@ public: /// a separate lock `EntryGuard::Lock`, it will make this part of code more coherent, /// but it will introduce one more mutex while it is avoidable. /// Introducing one more mutex just for coherency does not win the trade-off (isn't it?). - void setEvicting(bool evicting_, const LockedKey * locked_key, const CachePriorityGuard::Lock * lock) const + void setEvictingFlag(const LockedKey &, const CachePriorityGuard::Lock &) const { - if (evicting_ && (!locked_key || !lock)) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Setting evicting state to `true` can be done only under lock"); + auto prev = evicting.exchange(true, std::memory_order_relaxed); + chassert(!prev); + UNUSED(prev); + } - chassert(evicting.load() != evicting_); - evicting.store(evicting_); + void resetEvictingFlag() const + { + auto prev = evicting.exchange(false, std::memory_order_relaxed); + chassert(prev); + UNUSED(prev); } private: @@ -189,12 +193,9 @@ public: protected: IFileCachePriority(size_t max_size_, size_t max_elements_); - virtual void holdImpl( - size_t size, - size_t elements, - const CachePriorityGuard::Lock & lock); + virtual void holdImpl(size_t /* size */, size_t /* elements */, const CachePriorityGuard::Lock &) {} - virtual void releaseImpl(size_t size, size_t elements); + virtual void releaseImpl(size_t /* size */, size_t /* elements */) {} size_t max_size = 0; size_t max_elements = 0; diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 52be6368e49..7efc83c331a 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -50,12 +50,20 @@ struct FileSegmentMetadata : private boost::noncopyable return iterator->getEntry()->isEvicting(lock); } - void setEvicting(bool evicting, const LockedKey * locked_key, const CachePriorityGuard::Lock * lock) const + void setEvictingFlag(const LockedKey & locked_key, const CachePriorityGuard::Lock & lock) const { auto iterator = getQueueIterator(); if (!iterator) throw Exception(ErrorCodes::LOGICAL_ERROR, "Iterator is not set"); - iterator->getEntry()->setEvicting(evicting, locked_key, lock); + iterator->getEntry()->setEvictingFlag(locked_key, lock); + } + + void resetEvictingFlag() const + { + auto iterator = getQueueIterator(); + if (!iterator) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Iterator is not set"); + iterator->getEntry()->resetEvictingFlag(); } Priority::IteratorPtr getQueueIterator() const { return file_segment->getQueueIterator(); } From 0a5ce69880536c9125f5e990533b886c12a01d3e Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 26 Mar 2024 21:59:11 +0100 Subject: [PATCH 0683/1165] Review fix --- src/Interpreters/MutationsInterpreter.cpp | 2 +- src/Storages/MergeTree/checkDataPart.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 7d5443b3e65..35fd549559b 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -409,7 +409,7 @@ MutationsInterpreter::MutationsInterpreter( , available_columns(std::move(available_columns_)) , settings(std::move(settings_)) , select_limits(SelectQueryOptions().analyze(!settings.can_execute).ignoreLimits()) - , logger(getLogger("MutationsInterpreter")) + , logger(getLogger("MutationsInterpreter(" + source.getStorage()->getStorageID().getFullTableName() + ")")) { auto new_context = Context::createCopy(context_); if (new_context->getSettingsRef().allow_experimental_analyzer) diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index cabe06b0dde..d64568e0c3e 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -305,13 +305,13 @@ static IMergeTreeDataPart::Checksums checkDataPart( projections_on_disk.erase(projection_file); checksums_txt.remove(projection_file); - const auto exception = getCurrentExceptionMessage(true); + const auto exception_message = getCurrentExceptionMessage(true); if (!projection->is_broken) { LOG_WARNING(log, "Marking projection {} as broken ({}). Reason: {}", - name, projection_file, exception); - projection->setBrokenReason(exception, getCurrentExceptionCode()); + name, projection_file, exception_message); + projection->setBrokenReason(exception_message, getCurrentExceptionCode()); } if (throw_on_broken_projection) @@ -321,7 +321,7 @@ static IMergeTreeDataPart::Checksums checkDataPart( broken_projections_message += fmt::format( "Part {} has a broken projection {} (error: {})", - data_part->name, name, exception); + data_part->name, name, exception_message); } continue; From a0a7f23c219ab514ac9cc19f470295881c420b32 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 26 Mar 2024 22:08:38 +0100 Subject: [PATCH 0684/1165] Empty commit to restart checks From e5ba8e82a9fd705d0913e89848fa3e44880a2e80 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 Mar 2024 23:25:22 +0100 Subject: [PATCH 0685/1165] Tomorrow --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e576fb447c1..84e51c1efdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ # 2024 Changelog -### ClickHouse release 24.3 LTS, 2024-03-26 +### ClickHouse release 24.3 LTS, 2024-03-27 #### Upgrade Notes * The setting `allow_experimental_analyzer` is enabled by default and it switches the query analysis to a new implementation, which has better compatibility and feature completeness. The feature "analyzer" is considered beta instead of experimental. You can turn the old behavior by setting the `compatibility` to `24.2` or disabling the `allow_experimental_analyzer` setting. Watch the [video on YouTube](https://www.youtube.com/watch?v=zhrOYQpgvkk). From 4069ff6cb04a77690f8d9272171864e50b47e08c Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 26 Mar 2024 23:21:13 +0000 Subject: [PATCH 0686/1165] impl --- .../01193_metadata_loading.reference | 5 -- .../0_stateless/01193_metadata_loading.sh | 53 ------------------- 2 files changed, 58 deletions(-) delete mode 100644 tests/queries/0_stateless/01193_metadata_loading.reference delete mode 100755 tests/queries/0_stateless/01193_metadata_loading.sh diff --git a/tests/queries/0_stateless/01193_metadata_loading.reference b/tests/queries/0_stateless/01193_metadata_loading.reference deleted file mode 100644 index 9789cbf33ba..00000000000 --- a/tests/queries/0_stateless/01193_metadata_loading.reference +++ /dev/null @@ -1,5 +0,0 @@ -1000 0 2020-06-25 hello [1,2] [3,4] -1000 1 2020-06-26 word [10,20] [30,40] -ok -8000 0 2020-06-25 hello [1,2] [3,4] -8000 1 2020-06-26 word [10,20] [30,40] diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh deleted file mode 100755 index 69178a93d42..00000000000 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-parallel, no-fasttest, no-s3-storage, no-sanitize-coverage - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -# Check that attaching a database with a large number of tables is not too slow. -# it is the worst way of making performance test, nevertheless it can detect significant slowdown and some other issues, that usually found by stress test - -db="test_01193_$RANDOM_$RANDOM_$RANDOM_$RANDOM" -tables=1000 -threads=10 -count_multiplier=1 -max_time_ms=1500 - -create_tables() { - $CLICKHOUSE_CLIENT -q "WITH - 'CREATE TABLE $db.table_$1_' AS create1, - ' (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=' AS create2, - ['Memory', 'File(CSV)', 'Log', 'StripeLog', 'MergeTree ORDER BY i'] AS engines, - 'INSERT INTO $db.table_$1_' AS insert1, - ' VALUES (0, ''2020-06-25'', ''hello'', [1, 2], [3, 4]), (1, ''2020-06-26'', ''word'', [10, 20], [30, 40])' AS insert2 - SELECT arrayStringConcat( - groupArray( - create1 || toString(number) || create2 || engines[1 + number % length(engines)] || ';\n' || - insert1 || toString(number) || insert2 - ), ';\n') FROM numbers($tables) SETTINGS max_bytes_before_external_group_by = 0 FORMAT TSVRaw;" | $CLICKHOUSE_CLIENT -nm -} - -$CLICKHOUSE_CLIENT -q "CREATE DATABASE $db" - -for i in $(seq 1 $threads); do - create_tables "$i" & -done -wait - -$CLICKHOUSE_CLIENT -q "CREATE TABLE $db.table_merge (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=Merge('$db', '^table_')" -$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM merge('$db', '^table_9') GROUP BY i, d, s, n.i, n.f ORDER BY i" - -for i in {1..50}; do - $CLICKHOUSE_CLIENT -q "DETACH DATABASE $db" - $CLICKHOUSE_CLIENT --query_profiler_real_time_period_ns=100000000 --query_profiler_cpu_time_period_ns=100000000 -q "ATTACH DATABASE $db" --query_id="$db-$i"; -done - -$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS" -durations=$($CLICKHOUSE_CLIENT -q "SELECT groupArray(query_duration_ms) FROM system.query_log WHERE current_database = currentDatabase() AND query_id LIKE '$db-%' AND type=2") -$CLICKHOUSE_CLIENT -q "SELECT 'durations', '$db', $durations FORMAT Null" -$CLICKHOUSE_CLIENT -q "SELECT if(quantile(0.5)(arrayJoin($durations)) < $max_time_ms, 'ok', toString($durations))" - -$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" - -$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $db" From d5da446d2161aa957e80fb2983fc1698d159c689 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Wed, 27 Mar 2024 02:32:15 +0000 Subject: [PATCH 0687/1165] Tidy --- src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp | 2 +- src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp | 3 +-- src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp | 3 +-- src/Disks/ObjectStorages/Web/WebObjectStorage.cpp | 3 +-- src/Storages/StorageS3.cpp | 3 +-- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp index cbddc378a24..e717c88ed22 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp @@ -62,7 +62,7 @@ std::unique_ptr HDFSObjectStorage::readObjects( /// NOLI [this, disk_read_settings] (bool /* restricted_seek */, const StoredObject & object_) -> std::unique_ptr { - auto path = object_.remote_path; + const auto & path = object_.remote_path; size_t begin_of_path = path.find('/', path.find("//") + 2); auto hdfs_path = path.substr(begin_of_path); auto hdfs_uri = path.substr(0, begin_of_path); diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index cdd9cb62482..d44e17a0713 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -52,8 +52,7 @@ std::unique_ptr LocalObjectStorage::readObjects( /// NOL [=] (bool /* restricted_seek */, const StoredObject & object) -> std::unique_ptr { - auto & file_path = object.remote_path; - return createReadBufferFromFileBase(file_path, modified_settings, read_hint, file_size); + return createReadBufferFromFileBase(object.remote_path, modified_settings, read_hint, file_size); }; switch (read_settings.remote_fs_method) diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 52b5109ff8d..b343b73f7bd 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -174,11 +174,10 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT [this, settings_ptr, disk_read_settings] (bool restricted_seek, const StoredObject & object_) -> std::unique_ptr { - auto & path = object_.remote_path; return std::make_unique( client.get(), uri.bucket, - path, + object_.remote_path, uri.version_id, settings_ptr->request_settings, disk_read_settings, diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp index e0a82f702cb..69f6137cd2d 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp @@ -255,9 +255,8 @@ std::unique_ptr WebObjectStorage::readObject( /// NOLINT [this, read_settings, object_size] (bool /* restricted_seek */, const StoredObject & object_) -> std::unique_ptr { - auto & path_ = object_.remote_path; return std::make_unique( - fs::path(url) / path_, + fs::path(url) / object_.remote_path, getContext(), object_size, read_settings, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 0b985311864..2d3aef312bf 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -727,11 +727,10 @@ std::unique_ptr StorageS3Source::createAsyncS3ReadBuffer( [this, read_settings, object_size] (bool restricted_seek, const StoredObject & object) -> std::unique_ptr { - auto & path = object.remote_path; return std::make_unique( client, bucket, - path, + object.remote_path, version_id, request_settings, read_settings, From af2e86d72d3dde0636fef52068b720ff539e8e73 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 27 Mar 2024 10:21:16 +0100 Subject: [PATCH 0688/1165] make TSAN happy --- src/Storages/MaterializedView/RefreshSet.cpp | 2 +- src/Storages/MaterializedView/RefreshTask.cpp | 3 +-- src/Storages/MaterializedView/RefreshTask.h | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Storages/MaterializedView/RefreshSet.cpp b/src/Storages/MaterializedView/RefreshSet.cpp index 3d3407196e0..0125b958a41 100644 --- a/src/Storages/MaterializedView/RefreshSet.cpp +++ b/src/Storages/MaterializedView/RefreshSet.cpp @@ -81,7 +81,7 @@ void RefreshSet::emplace(StorageID id, const std::vector & dependenci throw Exception(ErrorCodes::LOGICAL_ERROR, "Refresh set entry already exists for table {}", id.getFullTableName()); addDependenciesLocked(id, dependencies); - task->setRefreshSetHandle(Handle(this, id, dependencies)); + task->setRefreshSetHandleUnlock(Handle(this, id, dependencies)); } void RefreshSet::addDependenciesLocked(const StorageID & id, const std::vector & dependencies) diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp index b77ab84f9a0..bc8cb0ce69a 100644 --- a/src/Storages/MaterializedView/RefreshTask.cpp +++ b/src/Storages/MaterializedView/RefreshTask.cpp @@ -509,9 +509,8 @@ std::chrono::system_clock::time_point RefreshTask::currentTime() const return std::chrono::system_clock::time_point(std::chrono::seconds(fake)); } -void RefreshTask::setRefreshSetHandle(RefreshSet::Handle && set_handle_) +void RefreshTask::setRefreshSetHandleUnlock(RefreshSet::Handle && set_handle_) { - std::lock_guard guard(mutex); set_handle = std::move(set_handle_); } diff --git a/src/Storages/MaterializedView/RefreshTask.h b/src/Storages/MaterializedView/RefreshTask.h index 4bd3c3e2843..1f050a97cd9 100644 --- a/src/Storages/MaterializedView/RefreshTask.h +++ b/src/Storages/MaterializedView/RefreshTask.h @@ -62,7 +62,7 @@ public: void setFakeTime(std::optional t); /// RefreshSet will set handle for refresh tasks, to avoid race condition. - void setRefreshSetHandle(RefreshSet::Handle && set_handle_); + void setRefreshSetHandleUnlock(RefreshSet::Handle && set_handle_); private: LoggerPtr log = nullptr; From af66e1f7c49e86017b0d71488b4509d450522a71 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:25:58 +0100 Subject: [PATCH 0689/1165] Fix style check --- src/Interpreters/Cache/IFileCachePriority.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index ddb1cf3c409..ff06f17ce36 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -14,11 +14,6 @@ namespace DB struct FileCacheReserveStat; class EvictionCandidates; -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - class IFileCachePriority : private boost::noncopyable { public: From 6732636222b3eedd6d0fe70358ba83083f84753b Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 27 Mar 2024 11:29:24 +0100 Subject: [PATCH 0690/1165] Add tests to the broken list --- tests/analyzer_tech_debt.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt index 1b65e3f5e38..89af93f581b 100644 --- a/tests/analyzer_tech_debt.txt +++ b/tests/analyzer_tech_debt.txt @@ -3,3 +3,8 @@ 02354_vector_search_queries 02901_parallel_replicas_rollup 02999_scalar_subqueries_bug_2 +# Flaky list +01825_type_json_in_array +01414_mutations_and_errors_zookeeper +# Check after ConstantNode refactoring +02154_parser_backtracking From 104774c646e17ac38e6affb0b92262fa284ff1d5 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 27 Mar 2024 11:39:33 +0100 Subject: [PATCH 0691/1165] Fix CI config --- tests/ci/test_ci_options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py index 61aeec59581..7c87144a84d 100644 --- a/tests/ci/test_ci_options.py +++ b/tests/ci/test_ci_options.py @@ -113,9 +113,9 @@ class TestCIOptions(unittest.TestCase): jobs_to_do, [ "Style check", - "Integration tests (asan, analyzer)", + "Integration tests (asan, old analyzer)", "package_release", - "Stateless tests (release, analyzer, s3, DatabaseReplicated)", + "Stateless tests (release, old analyzer, s3, DatabaseReplicated)", "package_asan", ], ) From 414bccc52763ced3ee8da50e45798400223a3523 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 27 Mar 2024 12:06:50 +0100 Subject: [PATCH 0692/1165] Revert "disallow LowCardinality input type for JSONExtract" (#61960) --- src/Functions/FunctionsJSON.h | 13 ++++++++++--- .../0_stateless/00918_json_functions.reference | 1 - tests/queries/0_stateless/00918_json_functions.sql | 2 -- .../02474_extract_fixedstring_from_json.reference | 7 ------- .../02474_extract_fixedstring_from_json.sql | 7 ------- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 8a193785f87..2539fa1aeb4 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -348,7 +348,6 @@ public: String getName() const override { return Name::name; } bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } - bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { @@ -470,6 +469,9 @@ public: else return_type = json_return_type; + /// Top-level LowCardinality columns are processed outside JSON parser. + json_return_type = removeLowCardinality(json_return_type); + DataTypes argument_types; argument_types.reserve(arguments.size()); for (const auto & argument : arguments) @@ -865,9 +867,11 @@ struct JSONExtractTree explicit LowCardinalityFixedStringNode(const size_t fixed_length_) : fixed_length(fixed_length_) { } bool insertResultToColumn(IColumn & dest, const Element & element) override { - // For types other than string, delegate the insertion to JSONExtractRawImpl. - if (!element.isString()) + // If element is an object we delegate the insertion to JSONExtractRawImpl + if (element.isObject()) return JSONExtractRawImpl::insertResultToLowCardinalityFixedStringColumn(dest, element, fixed_length); + else if (!element.isString()) + return false; auto str = element.getString(); if (str.size() > fixed_length) @@ -1482,6 +1486,9 @@ public: // We use insertResultToLowCardinalityFixedStringColumn in case we are inserting raw data in a Low Cardinality FixedString column static bool insertResultToLowCardinalityFixedStringColumn(IColumn & dest, const Element & element, size_t fixed_length) { + if (element.getObject().size() > fixed_length) + return false; + ColumnFixedString::Chars chars; WriteBufferFromVector buf(chars, AppendModeTag()); traverse(element, buf); diff --git a/tests/queries/0_stateless/00918_json_functions.reference b/tests/queries/0_stateless/00918_json_functions.reference index 7b725111755..43b15ded93d 100644 --- a/tests/queries/0_stateless/00918_json_functions.reference +++ b/tests/queries/0_stateless/00918_json_functions.reference @@ -69,7 +69,6 @@ hello (3333.6,'test') (3333.6333333333,'test') (3333.6333333333,'test') -\N 123456.1234 Decimal(20, 4) 123456.1234 Decimal(20, 4) 123456789012345.12 Decimal(30, 4) diff --git a/tests/queries/0_stateless/00918_json_functions.sql b/tests/queries/0_stateless/00918_json_functions.sql index 61fcb21fcbd..e19dd17670e 100644 --- a/tests/queries/0_stateless/00918_json_functions.sql +++ b/tests/queries/0_stateless/00918_json_functions.sql @@ -81,7 +81,6 @@ SELECT JSONExtract('{"a":3333.6333333333333333333333, "b":"test"}', 'Tuple(a Dec SELECT JSONExtract('{"a":"3333.6333333333333333333333", "b":"test"}', 'Tuple(a Decimal(10,1), b LowCardinality(String))'); SELECT JSONExtract('{"a":3333.6333333333333333333333, "b":"test"}', 'Tuple(a Decimal(20,10), b LowCardinality(String))'); SELECT JSONExtract('{"a":"3333.6333333333333333333333", "b":"test"}', 'Tuple(a Decimal(20,10), b LowCardinality(String))'); -SELECT JSONExtract(materialize('{"string_value":null}'), materialize('string_value'), 'LowCardinality(Nullable(String))'); SELECT JSONExtract('{"a":123456.123456}', 'a', 'Decimal(20, 4)') as a, toTypeName(a); SELECT JSONExtract('{"a":"123456.123456"}', 'a', 'Decimal(20, 4)') as a, toTypeName(a); SELECT JSONExtract('{"a":"123456789012345.12"}', 'a', 'Decimal(30, 4)') as a, toTypeName(a); @@ -327,4 +326,3 @@ SELECT JSONExtract('[]', JSONExtract('0', 'UInt256'), 'UInt256'); -- { serverErr SELECT '--show error: key of map type should be String'; SELECT JSONExtract('{"a": [100.0, 200], "b": [-100, 200.0, 300]}', 'Map(Int64, Array(Float64))'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT JSONExtract(materialize(toLowCardinality('{"string_value":null}')), materialize('string_value'), 'LowCardinality(Nullable(String))'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT } diff --git a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference index 21ddf5d3512..783d12fcf1a 100644 --- a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference +++ b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference @@ -8,10 +8,3 @@ \0\0\0\0\0 131231 131231 -1234 -1234 -{"b":131231} -\0\0\0\0 -1234567890 -18446744073709551615 --9223372036854775807 diff --git a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql index bbb9f55062b..cfc47e00cba 100644 --- a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql +++ b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql @@ -6,10 +6,3 @@ SELECT JSONExtract('{"a": 123456}', 'a', 'FixedString(5)'); SELECT JSONExtract('{"a": 123456}', 'a', 'FixedString(6)'); SELECT JSONExtract(materialize('{"a": 131231}'), 'a', 'LowCardinality(FixedString(5))') FROM numbers(2); SELECT JSONExtract(materialize('{"a": 131231}'), 'a', 'LowCardinality(FixedString(6))') FROM numbers(2); -SELECT JSONExtract(materialize('{"a": 131231, "b": 1234}'), 'b', 'LowCardinality(FixedString(4))'); -SELECT JSONExtract(materialize('{"a": 131231, "b": "1234"}'), 'b', 'LowCardinality(FixedString(4))'); -SELECT JSONExtract(materialize('{"a": {"b": 131231} }'), 'a', 'LowCardinality(FixedString(12))'); -SELECT JSONExtract(materialize('{"a": 131231, "b": 1234567890}'), 'b', 'LowCardinality(FixedString(4))'); -SELECT JSONExtract(materialize('{"a": 131231, "b": 1234567890}'), 'b', 'LowCardinality(FixedString(10))'); -SELECT JSONExtract(materialize('{"a": 18446744073709551615}'), 'a', 'LowCardinality(FixedString(20))'); -SELECT JSONExtract(materialize('{"a": -9223372036854775807}'), 'a', 'LowCardinality(FixedString(20))'); From 128c51c6fc6bd88db20b255a8183cb591b26c2ba Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 27 Mar 2024 11:27:30 +0000 Subject: [PATCH 0693/1165] Fix storage join loading order --- src/Disks/DiskLocal.cpp | 1 - src/Storages/StorageSet.cpp | 13 ++++++++++++- .../integration/test_async_load_databases/test.py | 3 +++ tests/integration/test_join_set_family_s3/test.py | 14 +++++++++++--- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index f5d67d37b07..1a8d46668e0 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -153,7 +153,6 @@ public: return dir_path / entry->path().filename(); } - String name() const override { return entry->path().filename(); } private: diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 7d7f3113cdb..8561c3f3aa8 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -247,6 +247,8 @@ void StorageSetOrJoinBase::restore() static const char * file_suffix = ".bin"; static const auto file_suffix_size = strlen(".bin"); + using FilePriority = std::pair; + std::priority_queue, std::greater> backup_files; for (auto dir_it{disk->iterateDirectory(path)}; dir_it->isValid(); dir_it->next()) { const auto & name = dir_it->name(); @@ -261,9 +263,18 @@ void StorageSetOrJoinBase::restore() if (file_num > increment) increment = file_num; - restoreFromFile(dir_it->path()); + backup_files.push({file_num, file_path}); } } + + /// Restore in the same order as blocks were written + /// It may be important for storage Join, user expect to get the first row (unless `join_any_take_last_row` setting is set) + /// but after restart we may have different order of blocks in memory. + while (!backup_files.empty()) + { + restoreFromFile(backup_files.top().second); + backup_files.pop(); + } } diff --git a/tests/integration/test_async_load_databases/test.py b/tests/integration/test_async_load_databases/test.py index 050b529a227..d06897b1045 100644 --- a/tests/integration/test_async_load_databases/test.py +++ b/tests/integration/test_async_load_databases/test.py @@ -122,6 +122,9 @@ def test_dependent_tables(started_cluster): ) query("create table system.join (n int, m int) engine=Join(any, left, n)") query("insert into system.join values (1, 1)") + for i in range(2, 100): + query(f"insert into system.join values (1, {i})") + query( "create table src (n int, m default joinGet('system.join', 'm', 1::int)," "t default dictGetOrNull('a.d', 'm', toUInt64(3))," diff --git a/tests/integration/test_join_set_family_s3/test.py b/tests/integration/test_join_set_family_s3/test.py index 38b56b7b15b..f0e1480d867 100644 --- a/tests/integration/test_join_set_family_s3/test.py +++ b/tests/integration/test_join_set_family_s3/test.py @@ -93,10 +93,18 @@ def test_join_s3(cluster): "CREATE TABLE testLocalJoin(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id)" ) node.query( - "CREATE TABLE testS3Join(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id) SETTINGS disk='s3'" + "CREATE TABLE testS3Join(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id) SETTINGS disk='s3', join_any_take_last_row = 1" ) node.query("INSERT INTO testLocalJoin VALUES (1, 'a')") + for i in range(1, 10): + c = chr(ord("a") + i) + node.query(f"INSERT INTO testLocalJoin VALUES (1, '{c}')") + + # because of `join_any_take_last_row = 1` we expect the last row with 'a' value + for i in range(1, 10): + c = chr(ord("a") + i) + node.query(f"INSERT INTO testS3Join VALUES (1, '{c}')") node.query("INSERT INTO testS3Join VALUES (1, 'a')") assert ( @@ -105,7 +113,7 @@ def test_join_s3(cluster): ) == "\t\na\ta\n\t\n" ) - assert_objects_count(cluster, 1) + assert_objects_count(cluster, 10) node.query("INSERT INTO testLocalJoin VALUES (2, 'b')") node.query("INSERT INTO testS3Join VALUES (2, 'b')") @@ -116,7 +124,7 @@ def test_join_s3(cluster): ) == "\t\na\ta\nb\tb\n" ) - assert_objects_count(cluster, 2) + assert_objects_count(cluster, 11) node.restart_clickhouse() assert ( From 5a40a871c80a18d387f44ecc63b110502fe44529 Mon Sep 17 00:00:00 2001 From: Geoff Genz Date: Wed, 27 Mar 2024 05:42:22 -0600 Subject: [PATCH 0694/1165] Fix Arrow format settings anchor --- docs/en/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index a76bb01ce9e..14db25a31e4 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -2356,7 +2356,7 @@ You can select data from a ClickHouse table and save them into some file in the $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filename.arrow} ``` -### Arrow format settings {#parquet-format-settings} +### Arrow format settings {#arrow-format-settings} - [output_format_arrow_low_cardinality_as_dictionary](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_low_cardinality_as_dictionary) - enable output ClickHouse LowCardinality type as Dictionary Arrow type. Default value - `false`. - [output_format_arrow_use_64_bit_indexes_for_dictionary](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_use_64_bit_indexes_for_dictionary) - use 64-bit integer type for Dictionary indexes. Default value - `false`. From 4ba1078314ae109683d43c48d03366964fcd74e4 Mon Sep 17 00:00:00 2001 From: ILya Limarenko <67974432+I-Limar@users.noreply.github.com> Date: Wed, 27 Mar 2024 13:10:14 +0100 Subject: [PATCH 0695/1165] Update url-functions.md suggestion to fix the example for the cutToFirstSignificantSubdomainWithWWW function --- docs/en/sql-reference/functions/url-functions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index ac81815b47f..a0b0170721c 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -128,9 +128,9 @@ Returns the part of the domain that includes top-level subdomains up to the “f For example: -- `cutToFirstSignificantSubdomain('https://news.clickhouse.com.tr/') = 'clickhouse.com.tr'`. -- `cutToFirstSignificantSubdomain('www.tr') = 'www.tr'`. -- `cutToFirstSignificantSubdomain('tr') = ''`. +- `cutToFirstSignificantSubdomainWithWWW('https://news.clickhouse.com.tr/') = 'clickhouse.com.tr'`. +- `cutToFirstSignificantSubdomainWithWWW('www.tr') = 'www.tr'`. +- `cutToFirstSignificantSubdomainWithWWW('tr') = ''`. ### cutToFirstSignificantSubdomainCustom From 2daaa660c53be9ee2ab683fcfa477505ff8fe44d Mon Sep 17 00:00:00 2001 From: Pablo Marcos Date: Wed, 27 Mar 2024 13:43:49 +0100 Subject: [PATCH 0696/1165] Update input_format_parquet_allow_missing_columns default value Default value for input_format_parquet_allow_missing_columns was modified from 0 to 1, but the documentation was never updated. --- docs/en/operations/settings/settings-formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 477fbf94625..a865a956b82 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -1367,7 +1367,7 @@ Default value: `1'000'000`. While importing data, when column is not found in schema default value will be used instead of error. -Disabled by default. +Enabled by default. ### input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference {#input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference} From f7bdbaaacff2bd9b3b010cef207efcd9b08f3d0f Mon Sep 17 00:00:00 2001 From: Arnaud Rocher Date: Wed, 27 Mar 2024 13:46:58 +0100 Subject: [PATCH 0697/1165] Fix section link casing --- docs/en/sql-reference/table-functions/generate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/table-functions/generate.md b/docs/en/sql-reference/table-functions/generate.md index 3b9b077af49..a78015e9830 100644 --- a/docs/en/sql-reference/table-functions/generate.md +++ b/docs/en/sql-reference/table-functions/generate.md @@ -53,7 +53,7 @@ SELECT * FROM random; └──────────────────────────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ ``` -In combination with [generateRandomStructure](../../sql-reference/functions/other-functions.md#generateRandomStructure): +In combination with [generateRandomStructure](../../sql-reference/functions/other-functions.md#generaterandomstructure): ```sql SELECT * FROM generateRandom(generateRandomStructure(4, 101), 101) LIMIT 3; From beecfec1b4fc67fae3f5062bef8dbe07a5b78fc8 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 27 Mar 2024 15:17:16 +0100 Subject: [PATCH 0698/1165] Update url-functions.md From d7187a172fec7ab691f7e7b05b2bcd55874aaea6 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 27 Mar 2024 15:24:16 +0000 Subject: [PATCH 0699/1165] CI: failover for job_url request from gh --- tests/ci/ci.py | 6 +++--- tests/ci/upload_result_helper.py | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 17552377971..3ba4e4845d7 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -798,7 +798,7 @@ class CiOptions: if match.startswith("job_"): if not res.ci_jobs: res.ci_jobs = [] - res.ci_jobs.append(match) + res.ci_jobs.append(match.removeprefix("job_")) elif match.startswith("ci_set_") and match in Labels: if not res.ci_sets: res.ci_sets = [] @@ -906,10 +906,10 @@ class CiOptions: # FIXME: to be removed in favor of include/exclude # 2. Handle "job_" tags if any if self.ci_jobs: - for tag in self.ci_jobs: + for job in self.ci_jobs: job_with_parents = CI_CONFIG.get_job_with_parents(job) print( - f"NOTE: CI Job's tag: [#job_{tag}], add jobs: [{job_with_parents}]" + f"NOTE: CI Job's tag: [#job_{job}], add jobs: [{job_with_parents}]" ) # always add requested job itself, even if it could be skipped jobs_to_do_requested.append(job_with_parents[0]) diff --git a/tests/ci/upload_result_helper.py b/tests/ci/upload_result_helper.py index 9dca3fae1dc..7404ebc6335 100644 --- a/tests/ci/upload_result_helper.py +++ b/tests/ci/upload_result_helper.py @@ -92,13 +92,19 @@ def upload_results( else: raw_log_url = GITHUB_JOB_URL() + try: + job_url = GITHUB_JOB_URL() + except Exception: + print("ERROR: Failed to get job URL from GH API, job report will use run URL instead.") + job_url = GITHUB_RUN_URL + if test_results or not ready_report_url: html_report = create_test_html_report( check_name, test_results, raw_log_url, GITHUB_RUN_URL, - GITHUB_JOB_URL(), + job_url, branch_url, branch_name, commit_url, From 42a27c5c875af506bddc8c4e932acf91ec0410a9 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 27 Mar 2024 16:27:47 +0100 Subject: [PATCH 0700/1165] Fix WriteBufferAzureBlobStorage destructor uncaught exception --- .../IO/WriteBufferFromAzureBlobStorage.cpp | 42 +++++++++++++++---- .../IO/WriteBufferFromAzureBlobStorage.h | 5 ++- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index 05b93dd1fa3..9432cdf9fef 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -65,7 +65,20 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( WriteBufferFromAzureBlobStorage::~WriteBufferFromAzureBlobStorage() { - finalize(); + LOG_TRACE(limitedLog, "Close WriteBufferFromAzureBlobStorage. {}.", blob_path); + + /// That destructor could be call with finalized=false in case of exceptions + if (!finalized) + { + LOG_INFO( + log, + "WriteBufferFromAzureBlobStorage is not finalized in destructor. " + "The file might not be written to AzureBlobStorage. " + "{}.", + blob_path); + } + + task_tracker->safeWaitAll(); } void WriteBufferFromAzureBlobStorage::execWithRetry(std::function func, size_t num_tries, size_t cost) @@ -102,9 +115,13 @@ void WriteBufferFromAzureBlobStorage::execWithRetry(std::function func, } } -void WriteBufferFromAzureBlobStorage::finalizeImpl() +void WriteBufferFromAzureBlobStorage::preFinalize() { - auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); + if (is_prefinalized) + return; + + // This function should not be run again + is_prefinalized = true; /// If there is only one block and size is less than or equal to max_single_part_upload_size /// then we use single part upload instead of multi part upload @@ -113,6 +130,7 @@ void WriteBufferFromAzureBlobStorage::finalizeImpl() size_t data_size = size_t(position() - memory.data()); if (data_size <= max_single_part_upload_size) { + auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); Azure::Core::IO::MemoryBodyStream memory_stream(reinterpret_cast(memory.data()), data_size); execWithRetry([&](){ block_blob_client.Upload(memory_stream); }, max_unexpected_write_error_retries, data_size); LOG_TRACE(log, "Committed single block for blob `{}`", blob_path); @@ -120,14 +138,22 @@ void WriteBufferFromAzureBlobStorage::finalizeImpl() } } + writePart(); +} - execWithRetry([this](){ next(); }, max_unexpected_write_error_retries); +void WriteBufferFromAzureBlobStorage::finalizeImpl() +{ + LOG_TRACE(log, "finalizeImpl WriteBufferFromAzureBlobStorage {}", blob_path); - task_tracker->waitAll(); + if (!is_prefinalized) + preFinalize(); - execWithRetry([&](){ block_blob_client.CommitBlockList(block_ids); }, max_unexpected_write_error_retries); - - LOG_TRACE(log, "Committed {} blocks for blob `{}`", block_ids.size(), blob_path); + if (!block_ids.empty()) + { + auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); + execWithRetry([&](){ block_blob_client.CommitBlockList(block_ids); }, max_unexpected_write_error_retries); + LOG_TRACE(log, "Committed {} blocks for blob `{}`", block_ids.size(), blob_path); + } } void WriteBufferFromAzureBlobStorage::nextImpl() diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index 6e10c07b255..7d4081ad792 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -41,7 +41,7 @@ public: ~WriteBufferFromAzureBlobStorage() override; void nextImpl() override; - + void preFinalize() override; std::string getFileName() const override { return blob_path; } void sync() override { next(); } @@ -65,6 +65,9 @@ private: const std::string blob_path; const WriteSettings write_settings; + /// Track that prefinalize() is called only once + bool is_prefinalized = false; + AzureClientPtr blob_container_client; std::vector block_ids; From a45439ed528fa557f77abf7cd250ddd44fdd364a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 27 Mar 2024 15:34:55 +0000 Subject: [PATCH 0701/1165] Automatic style fix --- tests/ci/upload_result_helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ci/upload_result_helper.py b/tests/ci/upload_result_helper.py index 7404ebc6335..cb745131e0d 100644 --- a/tests/ci/upload_result_helper.py +++ b/tests/ci/upload_result_helper.py @@ -95,7 +95,9 @@ def upload_results( try: job_url = GITHUB_JOB_URL() except Exception: - print("ERROR: Failed to get job URL from GH API, job report will use run URL instead.") + print( + "ERROR: Failed to get job URL from GH API, job report will use run URL instead." + ) job_url = GITHUB_RUN_URL if test_results or not ready_report_url: From d8b06588b8387f3aa30898914341b8a94166437d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 27 Mar 2024 20:14:52 +0300 Subject: [PATCH 0702/1165] Aggregator parallel merge cancellation --- .../AggregateFunctionUniq.h | 14 ++++---- .../Combinators/AggregateFunctionArray.h | 9 +++-- .../Combinators/AggregateFunctionIf.h | 9 +++-- .../Combinators/AggregateFunctionMerge.h | 9 +++-- .../Combinators/AggregateFunctionNull.h | 13 +++++-- .../Combinators/AggregateFunctionState.h | 9 +++-- src/AggregateFunctions/IAggregateFunction.h | 4 +-- src/AggregateFunctions/UniqExactSet.h | 19 +++++++--- src/Interpreters/Aggregator.cpp | 35 +++++++++++-------- src/Interpreters/Aggregator.h | 21 +++++++---- src/Processors/IProcessor.h | 4 +-- .../AggregatingInOrderTransform.cpp | 2 +- .../Transforms/AggregatingTransform.cpp | 22 ++++++++---- ...gingAggregatedMemoryEfficientTransform.cpp | 2 +- .../Transforms/MergingAggregatedTransform.cpp | 11 +++--- src/Processors/Transforms/RollupTransform.cpp | 2 +- 16 files changed, 122 insertions(+), 63 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionUniq.h b/src/AggregateFunctions/AggregateFunctionUniq.h index 891f2ac4284..c53b5e3bdb7 100644 --- a/src/AggregateFunctions/AggregateFunctionUniq.h +++ b/src/AggregateFunctions/AggregateFunctionUniq.h @@ -457,9 +457,9 @@ public: detail::Adder::add(this->data(place), columns, num_args, row_begin, row_end, flags, null_map); } - bool isParallelizeMergePrepareNeeded() const override { return is_parallelize_merge_prepare_needed;} + bool isParallelizeMergePrepareNeeded() const override { return is_parallelize_merge_prepare_needed; } - void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override { if constexpr (is_parallelize_merge_prepare_needed) { @@ -469,7 +469,7 @@ public: for (size_t i = 0; i < data_vec.size(); ++i) data_vec[i] = &this->data(places[i]).set; - DataSet::parallelizeMergePrepare(data_vec, thread_pool); + DataSet::parallelizeMergePrepare(data_vec, thread_pool, is_cancelled); } else { @@ -485,10 +485,10 @@ public: bool isAbleToParallelizeMerge() const override { return is_able_to_parallelize_merge; } bool canOptimizeEqualKeysRanges() const override { return !is_able_to_parallelize_merge; } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena *) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena *) const override { if constexpr (is_able_to_parallelize_merge) - this->data(place).set.merge(this->data(rhs).set, &thread_pool); + this->data(place).set.merge(this->data(rhs).set, &thread_pool, is_cancelled); else this->data(place).set.merge(this->data(rhs).set); } @@ -579,10 +579,10 @@ public: bool isAbleToParallelizeMerge() const override { return is_able_to_parallelize_merge; } bool canOptimizeEqualKeysRanges() const override { return !is_able_to_parallelize_merge; } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena *) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena *) const override { if constexpr (is_able_to_parallelize_merge) - this->data(place).set.merge(this->data(rhs).set, &thread_pool); + this->data(place).set.merge(this->data(rhs).set, &thread_pool, is_cancelled); else this->data(place).set.merge(this->data(rhs).set); } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h index 6b918926d0d..9dc5e274dab 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h @@ -144,9 +144,14 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override { - nested_func->merge(place, rhs, thread_pool, arena); + nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + { + nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version) const override diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h index df23398a10d..91dcfa4db0b 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h @@ -167,9 +167,14 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override { - nested_func->merge(place, rhs, thread_pool, arena); + nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + { + nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } void mergeBatch( diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h index 53c24bd60c1..5bc478116e0 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h @@ -113,9 +113,14 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override { - nested_func->merge(place, rhs, thread_pool, arena); + nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + { + nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version) const override diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h index 72ab3cf5acb..eef5f8bf66b 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h @@ -154,9 +154,18 @@ public: bool isAbleToParallelizeMerge() const override { return nested_function->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_function->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override { - nested_function->merge(nestedPlace(place), nestedPlace(rhs), thread_pool, arena); + AggregateDataPtrs nested_places(places.begin(), places.end()); + for (auto & nested_place : nested_places) + nested_place = nestedPlace(nested_place); + + nested_function->parallelizeMergePrepare(nested_places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + { + nested_function->merge(nestedPlace(place), nestedPlace(rhs), thread_pool, is_cancelled, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version) const override diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionState.h b/src/AggregateFunctions/Combinators/AggregateFunctionState.h index b0ab6d49604..7b2933d42c9 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionState.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionState.h @@ -94,9 +94,14 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override { - nested_func->merge(place, rhs, thread_pool, arena); + nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + { + nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version) const override diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 499185320e6..b33d4b20a16 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -151,7 +151,7 @@ public: virtual bool isParallelizeMergePrepareNeeded() const { return false; } - virtual void parallelizeMergePrepare(AggregateDataPtrs & /*places*/, ThreadPool & /*thread_pool*/) const + virtual void parallelizeMergePrepare(AggregateDataPtrs & /*places*/, ThreadPool & /*thread_pool*/, std::atomic * /*is_cancelled*/) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "parallelizeMergePrepare() with thread pool parameter isn't implemented for {} ", getName()); } @@ -168,7 +168,7 @@ public: /// Should be used only if isAbleToParallelizeMerge() returned true. virtual void - merge(AggregateDataPtr __restrict /*place*/, ConstAggregateDataPtr /*rhs*/, ThreadPool & /*thread_pool*/, Arena * /*arena*/) const + merge(AggregateDataPtr __restrict /*place*/, ConstAggregateDataPtr /*rhs*/, ThreadPool & /*thread_pool*/, std::atomic * /*is_cancelled*/, Arena * /*arena*/) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "merge() with thread pool parameter isn't implemented for {} ", getName()); } diff --git a/src/AggregateFunctions/UniqExactSet.h b/src/AggregateFunctions/UniqExactSet.h index 131c59b9ed6..18b0b830cb8 100644 --- a/src/AggregateFunctions/UniqExactSet.h +++ b/src/AggregateFunctions/UniqExactSet.h @@ -37,7 +37,7 @@ public: /// In merge, if one of the lhs and rhs is twolevelset and the other is singlelevelset, then the singlelevelset will need to convertToTwoLevel(). /// It's not in parallel and will cost extra large time if the thread_num is large. /// This method will convert all the SingleLevelSet to TwoLevelSet in parallel if the hashsets are not all singlelevel or not all twolevel. - static void parallelizeMergePrepare(const std::vector & data_vec, ThreadPool & thread_pool) + static void parallelizeMergePrepare(const std::vector & data_vec, ThreadPool & thread_pool, std::atomic * is_cancelled) { UInt64 single_level_set_num = 0; UInt64 all_single_hash_size = 0; @@ -63,7 +63,7 @@ public: try { auto data_vec_atomic_index = std::make_shared(0); - auto thread_func = [data_vec, data_vec_atomic_index, thread_group = CurrentThread::getGroup()]() + auto thread_func = [data_vec, data_vec_atomic_index, is_cancelled, thread_group = CurrentThread::getGroup()]() { SCOPE_EXIT_SAFE( if (thread_group) @@ -76,6 +76,9 @@ public: while (true) { + if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) + return; + const auto i = data_vec_atomic_index->fetch_add(1); if (i >= data_vec.size()) return; @@ -96,7 +99,7 @@ public: } } - auto merge(const UniqExactSet & other, ThreadPool * thread_pool = nullptr) + auto merge(const UniqExactSet & other, ThreadPool * thread_pool = nullptr, std::atomic * is_cancelled = nullptr) { if (isSingleLevel() && other.isTwoLevel()) convertToTwoLevel(); @@ -113,7 +116,12 @@ public: if (!thread_pool) { for (size_t i = 0; i < rhs.NUM_BUCKETS; ++i) + { + if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) + return; + lhs.impls[i].merge(rhs.impls[i]); + } } else { @@ -121,7 +129,7 @@ public: { auto next_bucket_to_merge = std::make_shared(0); - auto thread_func = [&lhs, &rhs, next_bucket_to_merge, thread_group = CurrentThread::getGroup()]() + auto thread_func = [&lhs, &rhs, next_bucket_to_merge, is_cancelled, thread_group = CurrentThread::getGroup()]() { SCOPE_EXIT_SAFE( if (thread_group) @@ -133,6 +141,9 @@ public: while (true) { + if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) + return; + const auto bucket = next_bucket_to_merge->fetch_add(1); if (bucket >= rhs.NUM_BUCKETS) return; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 7f3b961a598..837b4e47fba 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1435,13 +1435,14 @@ void NO_INLINE Aggregator::mergeOnIntervalWithoutKey( AggregatedDataVariants & data_variants, size_t row_begin, size_t row_end, - const AggregateColumnsConstData & aggregate_columns_data) const + const AggregateColumnsConstData & aggregate_columns_data, + std::atomic * is_cancelled) const { /// `data_variants` will destroy the states of aggregate functions in the destructor data_variants.aggregator = this; data_variants.init(AggregatedDataVariants::Type::without_key); - mergeWithoutKeyStreamsImpl(data_variants, row_begin, row_end, aggregate_columns_data); + mergeWithoutKeyStreamsImpl(data_variants, row_begin, row_end, aggregate_columns_data, is_cancelled); } @@ -2636,7 +2637,8 @@ void NO_INLINE Aggregator::mergeDataOnlyExistingKeysImpl( void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( - ManyAggregatedDataVariants & non_empty_data) const + ManyAggregatedDataVariants & non_empty_data, + std::atomic * is_cancelled) const { ThreadPool thread_pool{CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, params.max_threads}; @@ -2652,7 +2654,7 @@ void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( for (size_t result_num = 0; result_num < size; ++result_num) data_vec.emplace_back(non_empty_data[result_num]->without_key + offsets_of_aggregate_states[i]); - aggregate_functions[i]->parallelizeMergePrepare(data_vec, thread_pool); + aggregate_functions[i]->parallelizeMergePrepare(data_vec, thread_pool, is_cancelled); } } @@ -2668,6 +2670,7 @@ void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( res_data + offsets_of_aggregate_states[i], current_data + offsets_of_aggregate_states[i], thread_pool, + is_cancelled, res->aggregates_pool); else aggregate_functions[i]->merge( @@ -2954,17 +2957,19 @@ void NO_INLINE Aggregator::mergeStreamsImpl( void NO_INLINE Aggregator::mergeBlockWithoutKeyStreamsImpl( Block block, - AggregatedDataVariants & result) const + AggregatedDataVariants & result, + std::atomic * is_cancelled) const { AggregateColumnsConstData aggregate_columns = params.makeAggregateColumnsData(block); - mergeWithoutKeyStreamsImpl(result, 0, block.rows(), aggregate_columns); + mergeWithoutKeyStreamsImpl(result, 0, block.rows(), aggregate_columns, is_cancelled); } void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( AggregatedDataVariants & result, size_t row_begin, size_t row_end, - const AggregateColumnsConstData & aggregate_columns_data) const + const AggregateColumnsConstData & aggregate_columns_data, + std::atomic * is_cancelled) const { using namespace CurrentMetrics; @@ -2986,12 +2991,12 @@ void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( if (aggregate_functions[i]->isParallelizeMergePrepareNeeded()) { std::vector data_vec{res + offsets_of_aggregate_states[i], (*aggregate_columns_data[i])[row]}; - aggregate_functions[i]->parallelizeMergePrepare(data_vec, thread_pool); + aggregate_functions[i]->parallelizeMergePrepare(data_vec, thread_pool, is_cancelled); } if (aggregate_functions[i]->isAbleToParallelizeMerge()) aggregate_functions[i]->merge( - res + offsets_of_aggregate_states[i], (*aggregate_columns_data[i])[row], thread_pool, result.aggregates_pool); + res + offsets_of_aggregate_states[i], (*aggregate_columns_data[i])[row], thread_pool, is_cancelled, result.aggregates_pool); else aggregate_functions[i]->merge( res + offsets_of_aggregate_states[i], (*aggregate_columns_data[i])[row], result.aggregates_pool); @@ -3000,7 +3005,7 @@ void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( } -bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys) const +bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys, std::atomic * is_cancelled) const { /// `result` will destroy the states of aggregate functions in the destructor result.aggregator = this; @@ -3022,7 +3027,7 @@ bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool } if (result.type == AggregatedDataVariants::Type::without_key || block.info.is_overflows) - mergeBlockWithoutKeyStreamsImpl(std::move(block), result); + mergeBlockWithoutKeyStreamsImpl(std::move(block), result, is_cancelled); #define M(NAME, IS_TWO_LEVEL) \ else if (result.type == AggregatedDataVariants::Type::NAME) \ mergeStreamsImpl(std::move(block), result.aggregates_pool, *result.NAME, result.NAME->data, result.without_key, result.consecutive_keys_cache_stats, no_more_keys); @@ -3070,7 +3075,7 @@ bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool } -void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads) +void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads, std::atomic * is_cancelled) { if (bucket_to_blocks.empty()) return; @@ -3183,7 +3188,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari break; if (result.type == AggregatedDataVariants::Type::without_key || block.info.is_overflows) - mergeBlockWithoutKeyStreamsImpl(std::move(block), result); + mergeBlockWithoutKeyStreamsImpl(std::move(block), result, is_cancelled); #define M(NAME, IS_TWO_LEVEL) \ else if (result.type == AggregatedDataVariants::Type::NAME) \ @@ -3202,7 +3207,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari } -Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) +Block Aggregator::mergeBlocks(BlocksList & blocks, bool final, std::atomic * is_cancelled) { if (blocks.empty()) return {}; @@ -3264,7 +3269,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) bucket_num = -1; if (result.type == AggregatedDataVariants::Type::without_key || is_overflows) - mergeBlockWithoutKeyStreamsImpl(std::move(block), result); + mergeBlockWithoutKeyStreamsImpl(std::move(block), result, is_cancelled); #define M(NAME, IS_TWO_LEVEL) \ else if (result.type == AggregatedDataVariants::Type::NAME) \ diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index e339047063c..4bce700a099 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -266,7 +266,10 @@ public: AggregateFunctionInstruction * aggregate_instructions) const; /// Used for aggregate projection. - bool mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys) const; + bool mergeOnBlock(Block block, + AggregatedDataVariants & result, + bool & no_more_keys, + std::atomic * is_cancelled) const; void mergeOnBlockSmall( AggregatedDataVariants & result, @@ -279,7 +282,8 @@ public: AggregatedDataVariants & data_variants, size_t row_begin, size_t row_end, - const AggregateColumnsConstData & aggregate_columns_data) const; + const AggregateColumnsConstData & aggregate_columns_data, + std::atomic * is_cancelled) const; /** Convert the aggregation data structure into a block. * If overflow_row = true, then aggregates for rows that are not included in max_rows_to_group_by are put in the first block. @@ -294,13 +298,13 @@ public: using BucketToBlocks = std::map; /// Merge partially aggregated blocks separated to buckets into one data structure. - void mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads); + void mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads, std::atomic * is_cancelled); /// Merge several partially aggregated blocks into one. /// Precondition: for all blocks block.info.is_overflows flag must be the same. /// (either all blocks are from overflow data or none blocks are). /// The resulting block has the same value of is_overflows flag. - Block mergeBlocks(BlocksList & blocks, bool final); + Block mergeBlocks(BlocksList & blocks, bool final, std::atomic * is_cancelled); /** Split block with partially-aggregated data to many blocks, as if two-level method of aggregation was used. * This is needed to simplify merging of that data with other results, that are already two-level. @@ -486,7 +490,8 @@ private: Arena * arena) const; void mergeWithoutKeyDataImpl( - ManyAggregatedDataVariants & non_empty_data) const; + ManyAggregatedDataVariants & non_empty_data, + std::atomic * is_cancelled) const; template void mergeSingleLevelDataImpl( @@ -597,13 +602,15 @@ private: void mergeBlockWithoutKeyStreamsImpl( Block block, - AggregatedDataVariants & result) const; + AggregatedDataVariants & result, + std::atomic * is_cancelled) const; void mergeWithoutKeyStreamsImpl( AggregatedDataVariants & result, size_t row_begin, size_t row_end, - const AggregateColumnsConstData & aggregate_columns_data) const; + const AggregateColumnsConstData & aggregate_columns_data, + std::atomic * is_cancelled) const; template void mergeBucketImpl( diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h index c6bef186877..56b4509fe00 100644 --- a/src/Processors/IProcessor.h +++ b/src/Processors/IProcessor.h @@ -369,6 +369,8 @@ public: protected: virtual void onCancel() {} + std::atomic is_cancelled{false}; + private: /// For: /// - elapsed_us @@ -378,8 +380,6 @@ private: /// - output_wait_elapsed_us friend class ExecutingGraph; - std::atomic is_cancelled{false}; - std::string processor_description; /// For processors_profile_log diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index a39a0db1311..f959b2b01b4 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -160,7 +160,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) if (group_by_key) params->aggregator.mergeOnBlockSmall(variants, key_begin, key_end, aggregate_columns_data, key_columns_raw); else - params->aggregator.mergeOnIntervalWithoutKey(variants, key_begin, key_end, aggregate_columns_data); + params->aggregator.mergeOnIntervalWithoutKey(variants, key_begin, key_end, aggregate_columns_data, &is_cancelled); } else { diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index ea5c525d5f2..767448edc64 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -285,7 +285,11 @@ class ConvertingAggregatedToChunksTransform : public IProcessor { public: ConvertingAggregatedToChunksTransform(AggregatingTransformParamsPtr params_, ManyAggregatedDataVariantsPtr data_, size_t num_threads_) - : IProcessor({}, {params_->getHeader()}), params(std::move(params_)), data(std::move(data_)), num_threads(num_threads_) + : IProcessor({}, {params_->getHeader()}) + , params(std::move(params_)) + , data(std::move(data_)) + , shared_data(std::make_shared()) + , num_threads(num_threads_) { } @@ -346,8 +350,7 @@ public: for (auto & input : inputs) input.close(); - if (shared_data) - shared_data->is_cancelled.store(true); + shared_data->is_cancelled.store(true); return Status::Finished; } @@ -372,6 +375,11 @@ public: return prepareTwoLevel(); } + void onCancel() override + { + shared_data->is_cancelled.store(true, std::memory_order_seq_cst); + } + private: IProcessor::Status preparePushToOutput() { @@ -464,7 +472,7 @@ private: if (first->type == AggregatedDataVariants::Type::without_key || params->params.overflow_row) { - params->aggregator.mergeWithoutKeyDataImpl(*data); + params->aggregator.mergeWithoutKeyDataImpl(*data, &shared_data->is_cancelled); auto block = params->aggregator.prepareBlockAndFillWithoutKey( *first, params->final, first->type != AggregatedDataVariants::Type::without_key); @@ -506,7 +514,7 @@ private: void createSources() { AggregatedDataVariantsPtr & first = data->at(0); - shared_data = std::make_shared(); + processors.reserve(num_threads); for (size_t thread = 0; thread < num_threads; ++thread) { @@ -684,7 +692,7 @@ void AggregatingTransform::consume(Chunk chunk) { auto block = getInputs().front().getHeader().cloneWithColumns(chunk.detachColumns()); block = materializeBlock(block); - if (!params->aggregator.mergeOnBlock(block, variants, no_more_keys)) + if (!params->aggregator.mergeOnBlock(block, variants, no_more_keys, &is_cancelled)) is_consume_finished = true; } else @@ -704,7 +712,7 @@ void AggregatingTransform::initGenerate() if (variants.empty() && params->params.keys_size == 0 && !params->params.empty_result_for_aggregation_by_empty_set) { if (params->params.only_merge) - params->aggregator.mergeOnBlock(getInputs().front().getHeader(), variants, no_more_keys); + params->aggregator.mergeOnBlock(getInputs().front().getHeader(), variants, no_more_keys, &is_cancelled); else params->aggregator.executeOnBlock(getInputs().front().getHeader(), variants, key_columns, aggregate_columns, no_more_keys); } diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp index a92e2253314..3bfd7874ac7 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -363,7 +363,7 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk) res_info->chunk_num = chunks_to_merge->chunk_num; chunk.setChunkInfo(std::move(res_info)); - auto block = params->aggregator.mergeBlocks(blocks_list, params->final); + auto block = params->aggregator.mergeBlocks(blocks_list, params->final, &is_cancelled); if (!required_sort_description.empty()) sortBlock(block, required_sort_description); diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index e4955d06859..64207093568 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -39,11 +39,10 @@ void MergingAggregatedTransform::consume(Chunk chunk) if (const auto * agg_info = typeid_cast(info.get())) { /** If the remote servers used a two-level aggregation method, - * then blocks will contain information about the number of the bucket. - * Then the calculations can be parallelized by buckets. - * We decompose the blocks to the bucket numbers indicated in them. - */ - + * then blocks will contain information about the number of the bucket. + * Then the calculations can be parallelized by buckets. + * We decompose the blocks to the bucket numbers indicated in them. + */ auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns()); block.info.is_overflows = agg_info->is_overflows; block.info.bucket_num = agg_info->bucket_num; @@ -73,7 +72,7 @@ Chunk MergingAggregatedTransform::generate() next_block = blocks.begin(); /// TODO: this operation can be made async. Add async for IAccumulatingTransform. - params->aggregator.mergeBlocks(std::move(bucket_to_blocks), data_variants, max_threads); + params->aggregator.mergeBlocks(std::move(bucket_to_blocks), data_variants, max_threads, &is_cancelled); blocks = params->aggregator.convertToBlocks(data_variants, params->final, max_threads); next_block = blocks.begin(); } diff --git a/src/Processors/Transforms/RollupTransform.cpp b/src/Processors/Transforms/RollupTransform.cpp index a5d67fb2f15..20ee91a203a 100644 --- a/src/Processors/Transforms/RollupTransform.cpp +++ b/src/Processors/Transforms/RollupTransform.cpp @@ -57,7 +57,7 @@ Chunk GroupByModifierTransform::merge(Chunks && chunks, bool is_input, bool fina for (auto & chunk : chunks) blocks.emplace_back(header.cloneWithColumns(chunk.detachColumns())); - auto current_block = is_input ? params->aggregator.mergeBlocks(blocks, final) : output_aggregator->mergeBlocks(blocks, final); + auto current_block = is_input ? params->aggregator.mergeBlocks(blocks, final, &is_cancelled) : output_aggregator->mergeBlocks(blocks, final, &is_cancelled); auto num_rows = current_block.rows(); return Chunk(current_block.getColumns(), num_rows); } From c9749ae586b5778e1ae1f06bed5d79e5df6d3e41 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 27 Mar 2024 17:07:59 +0000 Subject: [PATCH 0703/1165] CI: remove unnecessary job url for Mark release ready --- tests/ci/mark_release_ready.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/ci/mark_release_ready.py b/tests/ci/mark_release_ready.py index 31415fef9c0..7ffb3c9a89b 100755 --- a/tests/ci/mark_release_ready.py +++ b/tests/ci/mark_release_ready.py @@ -5,7 +5,6 @@ import logging import os from commit_status_helper import get_commit, post_commit_status -from env_helper import GITHUB_JOB_URL from get_robot_token import get_best_robot_token from git_helper import commit as commit_arg from github_helper import GitHub @@ -33,7 +32,6 @@ def main(): help="if given, used instead of one from PRInfo", ) args = parser.parse_args() - url = "" description = "the release can be created from the commit, manually set" pr_info = None if not args.commit: @@ -41,7 +39,6 @@ def main(): if pr_info.event == pr_info.default_event: raise ValueError("neither launched from the CI nor commit is given") args.commit = pr_info.sha - url = GITHUB_JOB_URL() description = "the release can be created from the commit" args.token = args.token or get_best_robot_token() @@ -52,7 +49,7 @@ def main(): post_commit_status( commit, SUCCESS, - url, + "", description, RELEASE_READY_STATUS, pr_info, From 6a7d48b932f67e4191d280e17d1e18151489318b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 27 Mar 2024 17:26:12 +0000 Subject: [PATCH 0704/1165] Fix aliases. --- src/Analyzer/HashUtils.h | 16 +++++++++------- src/Analyzer/IQueryTreeNode.cpp | 4 ++-- src/Analyzer/IQueryTreeNode.h | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Analyzer/HashUtils.h b/src/Analyzer/HashUtils.h index 3727ea1ea14..8673361d499 100644 --- a/src/Analyzer/HashUtils.h +++ b/src/Analyzer/HashUtils.h @@ -11,35 +11,37 @@ namespace DB * Example of usage: * std::unordered_map map; */ -template +template struct QueryTreeNodeWithHash { QueryTreeNodeWithHash(QueryTreeNodePtrType node_) /// NOLINT : node(std::move(node_)) - , hash(node->getTreeHash()) + , hash(node->getTreeHash({.compare_aliases = compare_aliases})) {} QueryTreeNodePtrType node = nullptr; CityHash_v1_0_2::uint128 hash; }; -template -inline bool operator==(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) +template +inline bool operator==(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) { - return lhs.hash == rhs.hash && lhs.node->isEqual(*rhs.node); + return lhs.hash == rhs.hash && lhs.node->isEqual(*rhs.node, {.compare_aliases = compare_aliases}); } -template -inline bool operator!=(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) +template +inline bool operator!=(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) { return !(lhs == rhs); } using QueryTreeNodePtrWithHash = QueryTreeNodeWithHash; +using QueryTreeNodePtrWithHashWithoutAlias = QueryTreeNodeWithHash; using QueryTreeNodeRawPtrWithHash = QueryTreeNodeWithHash; using QueryTreeNodeConstRawPtrWithHash = QueryTreeNodeWithHash; using QueryTreeNodePtrWithHashSet = std::unordered_set; +using QueryTreeNodePtrWithHashWithoutAliasSet = std::unordered_set; using QueryTreeNodeConstRawPtrWithHashSet = std::unordered_set; template diff --git a/src/Analyzer/IQueryTreeNode.cpp b/src/Analyzer/IQueryTreeNode.cpp index d61cb0ffab1..7815b93c3ac 100644 --- a/src/Analyzer/IQueryTreeNode.cpp +++ b/src/Analyzer/IQueryTreeNode.cpp @@ -164,7 +164,7 @@ bool IQueryTreeNode::isEqual(const IQueryTreeNode & rhs, CompareOptions compare_ return true; } -IQueryTreeNode::Hash IQueryTreeNode::getTreeHash() const +IQueryTreeNode::Hash IQueryTreeNode::getTreeHash(CompareOptions compare_options) const { /** Compute tree hash with this node as root. * @@ -201,7 +201,7 @@ IQueryTreeNode::Hash IQueryTreeNode::getTreeHash() const } hash_state.update(static_cast(node_to_process->getNodeType())); - if (!node_to_process->alias.empty()) + if (compare_options.compare_aliases && !node_to_process->alias.empty()) { hash_state.update(node_to_process->alias.size()); hash_state.update(node_to_process->alias); diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index b07aa2d31b0..92e34616c4d 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -114,7 +114,7 @@ public: * Alias of query tree node is part of query tree hash. * Original AST is not part of query tree hash. */ - Hash getTreeHash() const; + Hash getTreeHash(CompareOptions compare_options = { .compare_aliases = true }) const; /// Get a deep copy of the query tree QueryTreeNodePtr clone() const; From 6127b0b89962d01b34ec4dd658afb21c7e7fd237 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 27 Mar 2024 17:43:27 +0000 Subject: [PATCH 0705/1165] Revert "Fix broken only_analyze in QueryAnalysisPass" This reverts commit c88cb6be9dc6dc9b877c2c30d1dd92d67e325b5f. --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 119 ++++++++++++---------- 1 file changed, 68 insertions(+), 51 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index a4893373d6c..afc51d27a0b 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2066,75 +2066,92 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden io.pipeline.setProgressCallback(context->getProgressCallback()); io.pipeline.setProcessListElement(context->getProcessListElement()); - Block block; - - while (block.rows() == 0 && executor.pull(block)) + if (only_analyze) { - } - - if (block.rows() == 0) - { - auto types = interpreter->getSampleBlock().getDataTypes(); - if (types.size() != 1) - types = {std::make_shared(types)}; - - auto & type = types[0]; - if (!type->isNullable()) + /// If query is only analyzed, then constants are not correct. + scalar_block = interpreter->getSampleBlock(); + for (auto & column : scalar_block) { - if (!type->canBeInsideNullable()) - throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, - "Scalar subquery returned empty result of type {} which cannot be Nullable", - type->getName()); - - type = makeNullable(type); + if (column.column->empty()) + { + auto mut_col = column.column->cloneEmpty(); + mut_col->insertDefault(); + column.column = std::move(mut_col); + } } - - auto scalar_column = type->createColumn(); - scalar_column->insert(Null()); - scalar_block.insert({std::move(scalar_column), type, "null"}); } else { - if (block.rows() != 1) - throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); + Block block; - Block tmp_block; - while (tmp_block.rows() == 0 && executor.pull(tmp_block)) + while (block.rows() == 0 && executor.pull(block)) { } - if (tmp_block.rows() != 0) - throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); - - block = materializeBlock(block); - size_t columns = block.columns(); - - if (columns == 1) + if (block.rows() == 0) { - auto & column = block.getByPosition(0); - /// Here we wrap type to nullable if we can. - /// It is needed cause if subquery return no rows, it's result will be Null. - /// In case of many columns, do not check it cause tuple can't be nullable. - if (!column.type->isNullable() && column.type->canBeInsideNullable()) + auto types = interpreter->getSampleBlock().getDataTypes(); + if (types.size() != 1) + types = {std::make_shared(types)}; + + auto & type = types[0]; + if (!type->isNullable()) { - column.type = makeNullable(column.type); - column.column = makeNullable(column.column); + if (!type->canBeInsideNullable()) + throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, + "Scalar subquery returned empty result of type {} which cannot be Nullable", + type->getName()); + + type = makeNullable(type); } - scalar_block = block; + auto scalar_column = type->createColumn(); + scalar_column->insert(Null()); + scalar_block.insert({std::move(scalar_column), type, "null"}); } else { - /** Make unique column names for tuple. - * - * Example: SELECT (SELECT 2 AS x, x) - */ - makeUniqueColumnNamesInBlock(block); + if (block.rows() != 1) + throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); - scalar_block.insert({ - ColumnTuple::create(block.getColumns()), - std::make_shared(block.getDataTypes(), block.getNames()), - "tuple"}); + Block tmp_block; + while (tmp_block.rows() == 0 && executor.pull(tmp_block)) + { + } + + if (tmp_block.rows() != 0) + throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); + + block = materializeBlock(block); + size_t columns = block.columns(); + + if (columns == 1) + { + auto & column = block.getByPosition(0); + /// Here we wrap type to nullable if we can. + /// It is needed cause if subquery return no rows, it's result will be Null. + /// In case of many columns, do not check it cause tuple can't be nullable. + if (!column.type->isNullable() && column.type->canBeInsideNullable()) + { + column.type = makeNullable(column.type); + column.column = makeNullable(column.column); + } + + scalar_block = block; + } + else + { + /** Make unique column names for tuple. + * + * Example: SELECT (SELECT 2 AS x, x) + */ + makeUniqueColumnNamesInBlock(block); + + scalar_block.insert({ + ColumnTuple::create(block.getColumns()), + std::make_shared(block.getDataTypes(), block.getNames()), + "tuple"}); + } } } From 4250c5fb19e4bb554ba21216c2eae4476f115406 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 27 Mar 2024 18:48:14 +0100 Subject: [PATCH 0706/1165] Update autogenerated version to 24.4.1.1 and contributors --- cmake/autogenerated_versions.txt | 10 +++---- .../StorageSystemContributors.generated.cpp | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 2929c64ded8..26cb0eb23c6 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54484) +SET(VERSION_REVISION 54485) SET(VERSION_MAJOR 24) -SET(VERSION_MINOR 3) +SET(VERSION_MINOR 4) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH 891689a41506d00aa169548f5b4a8774351242c4) -SET(VERSION_DESCRIBE v24.3.1.1-testing) -SET(VERSION_STRING 24.3.1.1) +SET(VERSION_GITHASH 2c5c589a882ceec35439650337b92db3e76f0081) +SET(VERSION_DESCRIBE v24.4.1.1-testing) +SET(VERSION_STRING 24.4.1.1) # end of autochange diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index 0ad22082863..6a66d8a7e19 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -245,6 +245,7 @@ const char * auto_contributors[] { "Brendan Cox", "Brett Hoerner", "Brian Hunter", + "Brokenice0415", "Bulat Gaifullin", "Camden Cheek", "Camilo Sierra", @@ -286,6 +287,7 @@ const char * auto_contributors[] { "Dale Mcdiarmid", "Dalitso Banda", "Dan Roscigno", + "Dan Wu", "DanRoscigno", "Dani Pozo", "Daniel Bershatsky", @@ -294,6 +296,7 @@ const char * auto_contributors[] { "Daniel Kutenin", "Daniel Pozo Escalona", "Daniel Qin", + "Daniil Ivanik", "Daniil Rubin", "Danila Kutenin", "Daniël van Eeden", @@ -634,6 +637,7 @@ const char * auto_contributors[] { "LiuCong", "LiuNeng", "LiuYangkuan", + "LiuYuan", "Lloyd-Pottiger", "Lopatin Konstantin", "Lorenzo Mangani", @@ -668,6 +672,7 @@ const char * auto_contributors[] { "Marek Vavruša", "Marek Vavruša", "Mariano Benítez Mulet", + "Marina Fathouat", "Mark Andreev", "Mark Frost", "Mark Needham", @@ -767,6 +772,7 @@ const char * auto_contributors[] { "N. Kolotov", "NIKITA MIKHAILOV", "Narek Galstyan", + "Nataly Merezhuk", "Natalya Chizhonkova", "Natasha Murashkina", "NeZeD [Mac Pro]", @@ -787,6 +793,7 @@ const char * auto_contributors[] { "Nikhil Raman", "Nikifor Seriakov", "Nikita", + "Nikita Fomichev", "Nikita Keba", "Nikita Lapkov", "Nikita Mikhailov", @@ -804,10 +811,12 @@ const char * auto_contributors[] { "Nikolay Degterinsky", "Nikolay Edigaryev", "Nikolay Kirsh", + "Nikolay Monkov", "Nikolay Semyachkin", "Nikolay Shcheglov", "Nikolay Vasiliev", "Nikolay Volosatov", + "Nikolay Yankin", "Nir Peled", "Nityananda Gohain", "Niu Zhaojie", @@ -831,11 +840,13 @@ const char * auto_contributors[] { "Orkhan Zeynalli", "Oskar Wojciski", "OuO", + "Oxide Computer Company", "PHO", "Pablo Alegre", "Pablo Marcos", "Pablo Musa", "Palash Goel", + "PapaToemmsn", "Paramtamtam", "Patrick Zippenfenig", "Paul Loyd", @@ -859,7 +870,9 @@ const char * auto_contributors[] { "Persiyanov Dmitriy Andreevich", "Pervakov Grigorii", "Pervakov Grigory", + "Peter", "Petr Vasilev", + "Pham Anh Tuan", "Philip Hallstrom", "Philippe Ombredanne", "PigInCloud", @@ -973,11 +986,14 @@ const char * auto_contributors[] { "SevaCode", "Seyed Mehrshad Hosseini", "Shane Andrade", + "Shanfeng Pang", "Shani Elharrar", "Shaun Struwig", "Sherry Wang", "Shoh Jahon", "Shri Bodas", + "Shuai li", + "Shubham Ranjan", "Sichen Zhao", "SiderZhang", "Sidorov Pavel", @@ -1139,6 +1155,7 @@ const char * auto_contributors[] { "Wangyang Guo", "Waterkin", "Weiqing Xu", + "William Schoeffel", "William Shallum", "Winter Zhang", "Xbitz29", @@ -1252,6 +1269,7 @@ const char * auto_contributors[] { "awesomeleo", "bakam412", "bbkas", + "beetelbrox", "benamazing", "benbiti", "bgranvea", @@ -1261,6 +1279,7 @@ const char * auto_contributors[] { "bkuschel", "blazerer", "bluebirddm", + "bluikko", "bo zeng", "bobrovskij artemij", "booknouse", @@ -1309,6 +1328,7 @@ const char * auto_contributors[] { "d.v.semenov", "dalei2019", "damozhaeva", + "danila-ermakov", "dankondr", "daoready", "darkkeks", @@ -1324,6 +1344,7 @@ const char * auto_contributors[] { "dheerajathrey", "dimarub2000", "dinosaur", + "divanik", "divanorama", "dkxiaohei", "dmi-feo", @@ -1454,6 +1475,7 @@ const char * auto_contributors[] { "joelynch", "johanngan", "johnnymatthews", + "josh-hildred", "jsc0218", "jthmath", "jun won", @@ -1595,6 +1617,7 @@ const char * auto_contributors[] { "nautaa", "ndchikin", "nellicus", + "nemonlou", "neng.liu", "never lee", "ni1l", @@ -1637,6 +1660,7 @@ const char * auto_contributors[] { "pufit", "pyos", "pzhdfy", + "qaziqarta", "qianlixiang", "qianmoQ", "qieqieplus", @@ -1684,8 +1708,10 @@ const char * auto_contributors[] { "sev7e0", "sevirov", "sfod", + "shabroo", "shangshujie", "shedx", + "shuai-xu", "shuchaome", "shuyang", "sichenzhao", @@ -1710,6 +1736,7 @@ const char * auto_contributors[] { "sundy-li", "sundyli", "sunlisheng", + "sunny", "sunny19930321", "svladykin", "tai", @@ -1733,6 +1760,7 @@ const char * auto_contributors[] { "tiger.yan", "timfursov", "tison", + "tomershafir", "tomtana", "topvisor", "tpanetti", @@ -1740,6 +1768,7 @@ const char * auto_contributors[] { "tyrionhuang", "ubuntu", "una", + "unashi", "unbyte", "unegare", "unknown", @@ -1882,6 +1911,7 @@ const char * auto_contributors[] { "董海镔", "袁焊忠", "谢磊", + "豪肥肥", "贾顺名(Jarvis)", "郭小龙", "陈小玉", From e849c48e21b13a698552a371acac8201b48bc802 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 27 Mar 2024 18:56:13 +0100 Subject: [PATCH 0707/1165] add a test --- src/Access/Common/AccessType.h | 2 +- src/Common/FailPoint.cpp | 4 +- src/Interpreters/InterpreterSystemQuery.cpp | 9 ++ src/Parsers/ASTSystemQuery.cpp | 1 + src/Parsers/ASTSystemQuery.h | 1 + src/Parsers/ParserSystemQuery.cpp | 1 + .../ReplicatedMergeTreeRestartingThread.cpp | 7 ++ .../MergeTree/ReplicatedMergeTreeSink.cpp | 5 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 + tests/integration/test_quorum_inserts/test.py | 82 +++++++++++++++++++ 10 files changed, 111 insertions(+), 3 deletions(-) diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 6095f8ce6d3..83b50ce96c3 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -205,7 +205,7 @@ enum class AccessType M(SYSTEM_FLUSH, "", GROUP, SYSTEM) \ M(SYSTEM_THREAD_FUZZER, "SYSTEM START THREAD FUZZER, SYSTEM STOP THREAD FUZZER, START THREAD FUZZER, STOP THREAD FUZZER", GLOBAL, SYSTEM) \ M(SYSTEM_UNFREEZE, "SYSTEM UNFREEZE", GLOBAL, SYSTEM) \ - M(SYSTEM_FAILPOINT, "SYSTEM ENABLE FAILPOINT, SYSTEM DISABLE FAILPOINT", GLOBAL, SYSTEM) \ + M(SYSTEM_FAILPOINT, "SYSTEM ENABLE FAILPOINT, SYSTEM DISABLE FAILPOINT, SYSTEM WAIT FAILPOINT", GLOBAL, SYSTEM) \ M(SYSTEM_LISTEN, "SYSTEM START LISTEN, SYSTEM STOP LISTEN", GLOBAL, SYSTEM) \ M(SYSTEM_JEMALLOC, "SYSTEM JEMALLOC PURGE, SYSTEM JEMALLOC ENABLE PROFILE, SYSTEM JEMALLOC DISABLE PROFILE, SYSTEM JEMALLOC FLUSH PROFILE", GLOBAL, SYSTEM) \ M(SYSTEM, "", GROUP, ALL) /* allows to execute SYSTEM {SHUTDOWN|RELOAD CONFIG|...} */ \ diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 9e551c8f2cd..2434c6004ad 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -50,7 +50,9 @@ static struct InitFiu REGULAR(check_table_query_delay_for_part) \ REGULAR(dummy_failpoint) \ REGULAR(prefetched_reader_pool_failpoint) \ - PAUSEABLE_ONCE(dummy_pausable_failpoint_once) \ + PAUSEABLE_ONCE(replicated_merge_tree_insert_retry_pause) \ + PAUSEABLE_ONCE(finish_set_quorum_failed_parts) \ + PAUSEABLE_ONCE(finish_clean_quorum_failed_parts) \ PAUSEABLE(dummy_pausable_failpoint) \ ONCE(execute_query_calling_empty_set_result_func_on_exception) diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 1cbc9c49631..5f8b0ae01c8 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -755,6 +755,14 @@ BlockIO InterpreterSystemQuery::execute() FailPointInjection::disableFailPoint(query.fail_point_name); break; } + case Type::WAIT_FAILPOINT: + { + getContext()->checkAccess(AccessType::SYSTEM_FAILPOINT); + LOG_TRACE(log, "waiting for failpoint {}", query.fail_point_name); + FailPointInjection::pauseFailPoint(query.fail_point_name); + LOG_TRACE(log, "finished failpoint {}", query.fail_point_name); + break; + } case Type::RESET_COVERAGE: { getContext()->checkAccess(AccessType::SYSTEM); @@ -1454,6 +1462,7 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() case Type::STOP_THREAD_FUZZER: case Type::START_THREAD_FUZZER: case Type::ENABLE_FAILPOINT: + case Type::WAIT_FAILPOINT: case Type::DISABLE_FAILPOINT: case Type::RESET_COVERAGE: case Type::UNKNOWN: diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 9215353e2b3..ed122b2b191 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -364,6 +364,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s } case Type::ENABLE_FAILPOINT: case Type::DISABLE_FAILPOINT: + case Type::WAIT_FAILPOINT: { settings.ostr << ' '; print_identifier(fail_point_name); diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 18a804ebc45..65c3f0eb328 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -87,6 +87,7 @@ public: UNFREEZE, ENABLE_FAILPOINT, DISABLE_FAILPOINT, + WAIT_FAILPOINT, SYNC_FILESYSTEM_CACHE, STOP_PULLING_REPLICATION_LOG, START_PULLING_REPLICATION_LOG, diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 81f9332c730..b660f947290 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -263,6 +263,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & } case Type::ENABLE_FAILPOINT: case Type::DISABLE_FAILPOINT: + case Type::WAIT_FAILPOINT: { ASTPtr ast; if (ParserIdentifier{}.parse(pos, ast, expected)) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index b79418da791..35f355d1d9b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,11 @@ namespace ErrorCodes extern const int REPLICA_IS_ALREADY_ACTIVE; } +namespace FailPoints +{ + extern const char finish_clean_quorum_failed_parts[]; +}; + /// Used to check whether it's us who set node `is_active`, or not. static String generateActiveNodeIdentifier() { @@ -241,6 +247,7 @@ void ReplicatedMergeTreeRestartingThread::removeFailedQuorumParts() storage.queue.removeFailedQuorumPart(part->info); } } + FailPointInjection::disableFailPoint(FailPoints::finish_clean_quorum_failed_parts); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 91f85cbd87d..7fcf6b971bb 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -30,6 +30,7 @@ namespace FailPoints extern const char replicated_merge_tree_commit_zk_fail_after_op[]; extern const char replicated_merge_tree_insert_quorum_fail_0[]; extern const char replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault[]; + extern const char replicated_merge_tree_insert_retry_pause[]; } namespace ErrorCodes @@ -945,6 +946,7 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: new_retry_controller.retryLoop([&] { fiu_do_on(FailPoints::replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault, { zookeeper->forceFailureBeforeOperation(); }); + FailPointInjection::pauseFailPoint(FailPoints::replicated_merge_tree_insert_retry_pause); zookeeper->setKeeper(storage.getZooKeeper()); node_exists = zookeeper->exists(fs::path(storage.replica_path) / "parts" / part->name); if (isQuorumEnabled()) @@ -956,7 +958,8 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: { LOG_INFO(log, "Part {} fails to commit and will not retry or clean garbage. Restarting Thread will do everything.", part->name); transaction.clear(); - return CommitRetryContext::ERROR; + /// `quorum/failed_parts/part_name` exists because table is read only for a while, So we return table is read only. + throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); } if (node_exists) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ce6735d9176..0ff5d4794e0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -141,6 +141,7 @@ namespace FailPoints { extern const char replicated_queue_fail_next_entry[]; extern const char replicated_queue_unfail_entries[]; + extern const char finish_set_quorum_failed_parts[]; } namespace ErrorCodes @@ -2221,6 +2222,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry, bool need_to_che if (code == Coordination::Error::ZOK) { LOG_DEBUG(log, "Marked quorum for part {} as failed.", entry.new_part_name); + FailPointInjection::disableFailPoint(FailPoints::finish_set_quorum_failed_parts); queue.removeFailedQuorumPart(part_info); return true; } diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py index 1276a6079f0..0cc53558521 100644 --- a/tests/integration/test_quorum_inserts/test.py +++ b/tests/integration/test_quorum_inserts/test.py @@ -1,7 +1,9 @@ +import concurrent import time import pytest from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) @@ -361,3 +363,83 @@ def test_insert_quorum_with_ttl(started_cluster): ) zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_ttl ON CLUSTER cluster") + +def test_insert_quorum_with_keeper_loss_connection(): + zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_keeper_fail ON CLUSTER cluster") + create_query = ( + "CREATE TABLE test_insert_quorum_with_keeper_loss" + "(a Int8, d Date) " + "Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') " + "ORDER BY a " + ) + + print("Create Replicated table with two replicas") + zero.query(create_query) + first.query(create_query) + + print("Stop fetches for test_insert_quorum_with_keeper_loss at first replica.") + first.query("SYSTEM STOP FETCHES test_insert_quorum_with_keeper_loss") + + print("Inject insert fail and retry pause for server zero") + zero.query("SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op") + zero.query("SYSTEM ENABLE FAILPOINT replicated_merge_tree_insert_retry_pause") + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + print("Inject zero") + insert_future = executor.submit( + lambda: zero.query( + "INSERT INTO test_insert_quorum_with_keeper_loss(a,d) VALUES(1, '2011-01-01')", + settings={"insert_quorum_timeout": 150000}, + ) + ) + + print("Inject zero keeper") + pm = PartitionManager() + pm.drop_instance_zk_connections(zero) + + print("Wait zero is not active") + retries = 0 + zk = cluster.get_kazoo_client("zoo1") + while True: + if ( + zk.exists( + "/clickhouse/tables/test_insert_quorum_with_keeper_loss/replicas/zero/is_active" + ) + is None + ): + break + print("replica is still active") + time.sleep(1) + retries += 1 + if retries == 120: + raise Exception("Can not wait cluster replica inactive") + + print("Inject first wait for quorum fail") + first.query("SYSTEM ENABLE FAILPOINT finish_set_quorum_failed_parts") + quorum_fail_future = executor.submit( + lambda: first.query("SYSTEM WAIT FAILPOINT finish_set_quorum_failed_parts", timeout=300) + ) + print("Start fetches at first node") + first.query("SYSTEM START FETCHES test_insert_quorum_with_keeper_loss") + + concurrent.futures.wait([quorum_fail_future]) + + assert(quorum_fail_future.exception() is None) + + print("Inject zero wait for clean quorum fail parts") + zero.query("SYSTEM ENABLE FAILPOINT finish_clean_quorum_failed_parts") + clean_quorum_fail_parts_future = executor.submit( + lambda: first.query("SYSTEM WAIT FAILPOINT finish_clean_quorum_failed_parts", timeout=300) + ) + print("Restore zero keeper") + pm.restore_instance_zk_connections(zero) + concurrent.futures.wait([clean_quorum_fail_parts_future]) + + assert(clean_quorum_fail_parts_future.exception() is None) + + print("Disable retry pause") + zero.query("SYSTEM DISABLE FAILPOINT replicated_merge_tree_insert_retry_pause") + concurrent.futures.wait([insert_future]) + assert(insert_future.exception() is not None) + assert(not zero.contains_in_log("LOGICAL_ERROR")) + assert(zero.contains_in_log("fails to commit and will not retry or clean garbage")) From 52417e3d42f5ef2cc51680435c63eaca6a4da47c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 27 Mar 2024 18:02:02 +0000 Subject: [PATCH 0708/1165] Update version_date.tsv and changelogs after v24.3.1.2672-lts --- SECURITY.md | 8 +- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v24.3.1.2672-lts.md | 537 +++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 1 + 6 files changed, 544 insertions(+), 8 deletions(-) create mode 100644 docs/changelogs/v24.3.1.2672-lts.md diff --git a/SECURITY.md b/SECURITY.md index 86578b188d8..4701f2ec70b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -13,18 +13,16 @@ The following versions of ClickHouse server are currently being supported with s | Version | Supported | |:-|:-| +| 24.3 | ✔️ | | 24.2 | ✔️ | | 24.1 | ✔️ | -| 23.12 | ✔️ | -| 23.11 | ❌ | -| 23.10 | ❌ | -| 23.9 | ❌ | +| 23.* | ❌ | | 23.8 | ✔️ | | 23.7 | ❌ | | 23.6 | ❌ | | 23.5 | ❌ | | 23.4 | ❌ | -| 23.3 | ✔️ | +| 23.3 | ❌ | | 23.2 | ❌ | | 23.1 | ❌ | | 22.* | ❌ | diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 17eee6d4287..19d90fa509e 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.2.2.71" +ARG VERSION="24.3.1.2672" ARG PACKAGES="clickhouse-keeper" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index bd5fa313adc..ee93b6eb8f0 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.2.2.71" +ARG VERSION="24.3.1.2672" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 256dcdc029f..e92823b686a 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -27,7 +27,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="24.2.2.71" +ARG VERSION="24.3.1.2672" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v24.3.1.2672-lts.md b/docs/changelogs/v24.3.1.2672-lts.md new file mode 100644 index 00000000000..e5d008680a8 --- /dev/null +++ b/docs/changelogs/v24.3.1.2672-lts.md @@ -0,0 +1,537 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.3.1.2672-lts (2c5c589a882) FIXME as compared to v24.2.1.2248-stable (891689a4150) + +#### Backward Incompatible Change +* Don't allow to set max_parallel_replicas to 0 as it doesn't make sense. Setting it to 0 could lead to unexpected logical errors. Closes [#60140](https://github.com/ClickHouse/ClickHouse/issues/60140). [#60430](https://github.com/ClickHouse/ClickHouse/pull/60430) ([Kruglov Pavel](https://github.com/Avogar)). +* Change the column name from `duration_ms` to `duration_microseconds` in the `system.zookeeper` table to reflect the reality that the duration is in the microsecond resolution. [#60774](https://github.com/ClickHouse/ClickHouse/pull/60774) ([Duc Canh Le](https://github.com/canhld94)). +* Reject incoming INSERT queries in case when query-level settings `async_insert` and `deduplicate_blocks_in_dependent_materialized_views` are enabled together. This behaviour is controlled by a setting `throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert` and enabled by default. This is a continuation of https://github.com/ClickHouse/ClickHouse/pull/59699 needed to unblock https://github.com/ClickHouse/ClickHouse/pull/59915. [#60888](https://github.com/ClickHouse/ClickHouse/pull/60888) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Utility `clickhouse-copier` is moved to a separate repository on GitHub: https://github.com/ClickHouse/copier. It is no longer included in the bundle but is still available as a separate download. This closes: [#60734](https://github.com/ClickHouse/ClickHouse/issues/60734) This closes: [#60540](https://github.com/ClickHouse/ClickHouse/issues/60540) This closes: [#60250](https://github.com/ClickHouse/ClickHouse/issues/60250) This closes: [#52917](https://github.com/ClickHouse/ClickHouse/issues/52917) This closes: [#51140](https://github.com/ClickHouse/ClickHouse/issues/51140) This closes: [#47517](https://github.com/ClickHouse/ClickHouse/issues/47517) This closes: [#47189](https://github.com/ClickHouse/ClickHouse/issues/47189) This closes: [#46598](https://github.com/ClickHouse/ClickHouse/issues/46598) This closes: [#40257](https://github.com/ClickHouse/ClickHouse/issues/40257) This closes: [#36504](https://github.com/ClickHouse/ClickHouse/issues/36504) This closes: [#35485](https://github.com/ClickHouse/ClickHouse/issues/35485) This closes: [#33702](https://github.com/ClickHouse/ClickHouse/issues/33702) This closes: [#26702](https://github.com/ClickHouse/ClickHouse/issues/26702) ### Documentation entry for user-facing changes. [#61058](https://github.com/ClickHouse/ClickHouse/pull/61058) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* To increase compatibility with MySQL, function `locate` now accepts arguments `(needle, haystack[, start_pos])` by default. The previous behavior `(haystack, needle, [, start_pos])` can be restored by setting `function_locate_has_mysql_compatible_argument_order = 0`. [#61092](https://github.com/ClickHouse/ClickHouse/pull/61092) ([Robert Schulze](https://github.com/rschu1ze)). +* The obsolete in-memory data parts have been deprecated since version 23.5 and have not been supported since version 23.10. Now the remaining code is removed. Continuation of [#55186](https://github.com/ClickHouse/ClickHouse/issues/55186) and [#45409](https://github.com/ClickHouse/ClickHouse/issues/45409). It is unlikely that you have used in-memory data parts because they were available only before version 23.5 and only when you enabled them manually by specifying the corresponding SETTINGS for a MergeTree table. To check if you have in-memory data parts, run the following query: `SELECT part_type, count() FROM system.parts GROUP BY part_type ORDER BY part_type`. To disable the usage of in-memory data parts, do `ALTER TABLE ... MODIFY SETTING min_bytes_for_compact_part = DEFAULT, min_rows_for_compact_part = DEFAULT`. Before upgrading from old ClickHouse releases, first check that you don't have in-memory data parts. If there are in-memory data parts, disable them first, then wait while there are no in-memory data parts and continue the upgrade. [#61127](https://github.com/ClickHouse/ClickHouse/pull/61127) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Forbid `SimpleAggregateFunction` in `ORDER BY` of `MergeTree` tables (like `AggregateFunction` is forbidden, but they are forbidden because they are not comparable) by default (use `allow_suspicious_primary_key` to allow them). [#61399](https://github.com/ClickHouse/ClickHouse/pull/61399) ([Azat Khuzhin](https://github.com/azat)). +* ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. This is controlled by the settings, `output_format_parquet_string_as_string`, `output_format_orc_string_as_string`, `output_format_arrow_string_as_string`. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases. Parquet/ORC/Arrow supports many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools lack support for the faster `lz4` compression method, that's why we set `zstd` by default. This is controlled by the settings `output_format_parquet_compression_method`, `output_format_orc_compression_method`, and `output_format_arrow_compression_method`. We changed the default to `zstd` for Parquet and ORC, but not Arrow (it is emphasized for low-level usages). [#61817](https://github.com/ClickHouse/ClickHouse/pull/61817) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In the new ClickHouse version, the functions `geoDistance`, `greatCircleDistance`, and `greatCircleAngle` will use 64-bit double precision floating point data type for internal calculations and return type if all the arguments are Float64. This closes [#58476](https://github.com/ClickHouse/ClickHouse/issues/58476). In previous versions, the function always used Float32. You can switch to the old behavior by setting `geo_distance_returns_float64_on_float64_arguments` to `false` or setting `compatibility` to `24.2` or earlier. [#61848](https://github.com/ClickHouse/ClickHouse/pull/61848) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Topk/topkweighed support mode, which return count of values and it's error. [#54508](https://github.com/ClickHouse/ClickHouse/pull/54508) ([UnamedRus](https://github.com/UnamedRus)). +* Add generate_series as a table function. This function generates table with an arithmetic progression with natural numbers. [#59390](https://github.com/ClickHouse/ClickHouse/pull/59390) ([divanik](https://github.com/divanik)). +* Support reading and writing backups as tar archives. [#59535](https://github.com/ClickHouse/ClickHouse/pull/59535) ([josh-hildred](https://github.com/josh-hildred)). +* Implemented support for S3Express buckets. [#59965](https://github.com/ClickHouse/ClickHouse/pull/59965) ([Nikita Taranov](https://github.com/nickitat)). +* Allow to attach parts from a different disk * attach partition from the table on other disks using copy instead of hard link (such as instant table) * attach partition using copy when the hard link fails even on the same disk. [#60112](https://github.com/ClickHouse/ClickHouse/pull/60112) ([Unalian](https://github.com/Unalian)). +* Added function `toMillisecond` which returns the millisecond component for values of type`DateTime` or `DateTime64`. [#60281](https://github.com/ClickHouse/ClickHouse/pull/60281) ([Shaun Struwig](https://github.com/Blargian)). +* Make all format names case insensitive, like Tsv, or TSV, or tsv, or even rowbinary. [#60420](https://github.com/ClickHouse/ClickHouse/pull/60420) ([豪肥肥](https://github.com/HowePa)). +* Add four properties to the `StorageMemory` (memory-engine) `min_bytes_to_keep, max_bytes_to_keep, min_rows_to_keep` and `max_rows_to_keep` - Add tests to reflect new changes - Update `memory.md` documentation - Add table `context` property to `MemorySink` to enable access to table parameter bounds. [#60612](https://github.com/ClickHouse/ClickHouse/pull/60612) ([Jake Bamrah](https://github.com/JakeBamrah)). +* Added function `toMillisecond` which returns the millisecond component for values of type`DateTime` or `DateTime64`. [#60649](https://github.com/ClickHouse/ClickHouse/pull/60649) ([Robert Schulze](https://github.com/rschu1ze)). +* Separate limits on number of waiting and executing queries. Added new server setting `max_waiting_queries` that limits the number of queries waiting due to `async_load_databases`. Existing limits on number of executing queries no longer count waiting queries. [#61053](https://github.com/ClickHouse/ClickHouse/pull/61053) ([Sergei Trifonov](https://github.com/serxa)). +* Add support for `ATTACH PARTITION ALL`. [#61107](https://github.com/ClickHouse/ClickHouse/pull/61107) ([Kirill Nikiforov](https://github.com/allmazz)). +* Add a new function, `getClientHTTPHeader`. This closes [#54665](https://github.com/ClickHouse/ClickHouse/issues/54665). Co-authored with @lingtaolf. [#61820](https://github.com/ClickHouse/ClickHouse/pull/61820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Performance Improvement +* Improve the performance of serialized aggregation method when involving multiple [nullable] columns. This is a general version of [#51399](https://github.com/ClickHouse/ClickHouse/issues/51399) that doesn't compromise on abstraction integrity. [#55809](https://github.com/ClickHouse/ClickHouse/pull/55809) ([Amos Bird](https://github.com/amosbird)). +* Lazy build join output to improve performance of ALL join. [#58278](https://github.com/ClickHouse/ClickHouse/pull/58278) ([LiuNeng](https://github.com/liuneng1994)). +* Improvements to aggregate functions ArgMin / ArgMax / any / anyLast / anyHeavy, as well as `ORDER BY {u8/u16/u32/u64/i8/i16/u32/i64) LIMIT 1` queries. [#58640](https://github.com/ClickHouse/ClickHouse/pull/58640) ([Raúl Marín](https://github.com/Algunenano)). +* Trivial optimize on column filter. Avoid those filter columns whoes underlying data type is not number being filtered with `result_size_hint = -1`. Peak memory can be reduced to 44% of the original in some cases. [#59698](https://github.com/ClickHouse/ClickHouse/pull/59698) ([李扬](https://github.com/taiyang-li)). +* If the table's primary key contains mostly useless columns, don't keep them in memory. This is controlled by a new setting `primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns` with the value `0.9` by default, which means: for a composite primary key, if a column changes its value for at least 0.9 of all the times, the next columns after it will be not loaded. [#60255](https://github.com/ClickHouse/ClickHouse/pull/60255) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Execute multiIf function columnarly when result_type's underlying type is number. [#60384](https://github.com/ClickHouse/ClickHouse/pull/60384) ([李扬](https://github.com/taiyang-li)). +* Faster (almost 2x) mutexes (was slower due to ThreadFuzzer). [#60823](https://github.com/ClickHouse/ClickHouse/pull/60823) ([Azat Khuzhin](https://github.com/azat)). +* Move connection drain from prepare to work, and drain multiple connections in parallel. [#60845](https://github.com/ClickHouse/ClickHouse/pull/60845) ([lizhuoyu5](https://github.com/lzydmxy)). +* Optimize insertManyFrom of nullable number or nullable string. [#60846](https://github.com/ClickHouse/ClickHouse/pull/60846) ([李扬](https://github.com/taiyang-li)). +* Optimized function `dotProduct` to omit unnecessary and expensive memory copies. [#60928](https://github.com/ClickHouse/ClickHouse/pull/60928) ([Robert Schulze](https://github.com/rschu1ze)). +* Operations with the filesystem cache will suffer less from the lock contention. [#61066](https://github.com/ClickHouse/ClickHouse/pull/61066) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Optimize ColumnString::replicate and prevent memcpySmallAllowReadWriteOverflow15Impl from being optimized to built-in memcpy. Close [#61074](https://github.com/ClickHouse/ClickHouse/issues/61074). ColumnString::replicate speeds up by 2.46x on x86-64. [#61075](https://github.com/ClickHouse/ClickHouse/pull/61075) ([李扬](https://github.com/taiyang-li)). +* 30x faster printing for 256-bit integers. [#61100](https://github.com/ClickHouse/ClickHouse/pull/61100) ([Raúl Marín](https://github.com/Algunenano)). +* If a query with a syntax error contained COLUMNS matcher with a regular expression, the regular expression was compiled each time during the parser's backtracking, instead of being compiled once. This was a fundamental error. The compiled regexp was put to AST. But the letter A in AST means "abstract" which means it should not contain heavyweight objects. Parts of AST can be created and discarded during parsing, including a large number of backtracking. This leads to slowness on the parsing side and consequently allows DoS by a readonly user. But the main problem is that it prevents progress in fuzzers. [#61543](https://github.com/ClickHouse/ClickHouse/pull/61543) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a new analyzer pass to optimize in single value. [#61564](https://github.com/ClickHouse/ClickHouse/pull/61564) ([LiuNeng](https://github.com/liuneng1994)). + +#### Improvement +* While running the MODIFY COLUMN query for materialized views, check the inner table's structure to ensure every column exists. [#47427](https://github.com/ClickHouse/ClickHouse/pull/47427) ([sunny](https://github.com/sunny19930321)). +* Added table `system.keywords` which contains all the keywords from parser. Mostly needed and will be used for better fuzzing and syntax highlighting. [#51808](https://github.com/ClickHouse/ClickHouse/pull/51808) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Ordinary database engine is deprecated. You will receive a warning in clickhouse-client if your server is using it. This closes [#52229](https://github.com/ClickHouse/ClickHouse/issues/52229). [#56942](https://github.com/ClickHouse/ClickHouse/pull/56942) ([shabroo](https://github.com/shabroo)). +* All zero copy locks related to a table have to be dropped when the table is dropped. The directory which contains these locks has to be removed also. [#57575](https://github.com/ClickHouse/ClickHouse/pull/57575) ([Sema Checherinda](https://github.com/CheSema)). +* Allow declaring enum in external table structure. [#57857](https://github.com/ClickHouse/ClickHouse/pull/57857) ([Duc Canh Le](https://github.com/canhld94)). +* Consider lightweight deleted rows when selecting parts to merge. [#58223](https://github.com/ClickHouse/ClickHouse/pull/58223) ([Zhuo Qiu](https://github.com/jewelzqiu)). +* This PR makes http/https connections reusable for all uses cases. Even when response is 3xx or 4xx. [#58845](https://github.com/ClickHouse/ClickHouse/pull/58845) ([Sema Checherinda](https://github.com/CheSema)). +* Added comments for columns for more system tables. Continuation of https://github.com/ClickHouse/ClickHouse/pull/58356. [#59016](https://github.com/ClickHouse/ClickHouse/pull/59016) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Now we can use virtual columns in PREWHERE. It's worthwhile for non-const virtual columns like `_part_offset`. [#59033](https://github.com/ClickHouse/ClickHouse/pull/59033) ([Amos Bird](https://github.com/amosbird)). +* Add ability to skip read-only replicas for INSERT into Distributed engine (Controlled with `distributed_insert_skip_read_only_replicas` setting, by default OFF - backward compatible). [#59176](https://github.com/ClickHouse/ClickHouse/pull/59176) ([Azat Khuzhin](https://github.com/azat)). +* Instead using a constant key, now object storage generates key for determining remove objects capability. [#59495](https://github.com/ClickHouse/ClickHouse/pull/59495) ([Sema Checherinda](https://github.com/CheSema)). +* Add positional pread in libhdfs3. If you want to call positional read in libhdfs3, use the hdfsPread function in hdfs.h as follows. `tSize hdfsPread(hdfsFS fs, hdfsFile file, void * buffer, tSize length, tOffset position);`. [#59624](https://github.com/ClickHouse/ClickHouse/pull/59624) ([M1eyu](https://github.com/M1eyu2018)). +* Add asynchronous WriteBuffer for AzureBlobStorage similar to S3. [#59929](https://github.com/ClickHouse/ClickHouse/pull/59929) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Allow "local" as object storage type instead of "local_blob_storage". [#60165](https://github.com/ClickHouse/ClickHouse/pull/60165) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improved overall usability of virtual columns. Now it is allowed to use virtual columns in `PREWHERE` (it's worthwhile for non-const virtual columns like `_part_offset`). Now a builtin documentation is available for virtual columns as a comment of column in `DESCRIBE` query with enabled setting `describe_include_virtual_columns`. [#60205](https://github.com/ClickHouse/ClickHouse/pull/60205) ([Anton Popov](https://github.com/CurtizJ)). +* Parallel flush of pending INSERT blocks of Distributed engine on `DETACH`/server shutdown and `SYSTEM FLUSH DISTRIBUTED` (Parallelism will work only if you have multi disk policy for table (like everything in Distributed engine right now)). [#60225](https://github.com/ClickHouse/ClickHouse/pull/60225) ([Azat Khuzhin](https://github.com/azat)). +* Filter setting is improper in `joinRightColumnsSwitchNullability`, resolve [#59625](https://github.com/ClickHouse/ClickHouse/issues/59625). [#60259](https://github.com/ClickHouse/ClickHouse/pull/60259) ([lgbo](https://github.com/lgbo-ustc)). +* Add a setting to force read-through cache for merges. [#60308](https://github.com/ClickHouse/ClickHouse/pull/60308) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Issue [#57598](https://github.com/ClickHouse/ClickHouse/issues/57598) mentions a variant behaviour regarding transaction handling. An issued COMMIT/ROLLBACK when no transaction is active is reported as an error contrary to MySQL behaviour. [#60338](https://github.com/ClickHouse/ClickHouse/pull/60338) ([PapaToemmsn](https://github.com/PapaToemmsn)). +* Added `none_only_active` mode for `distributed_ddl_output_mode` setting. [#60340](https://github.com/ClickHouse/ClickHouse/pull/60340) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Allow configuring HTTP redirect handlers for clickhouse-server. For example, you can make `/` redirect to the Play UI. [#60390](https://github.com/ClickHouse/ClickHouse/pull/60390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The advanced dashboard has slightly better colors for multi-line graphs. [#60391](https://github.com/ClickHouse/ClickHouse/pull/60391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Function `substring` now has a new alias `byteSlice`. [#60494](https://github.com/ClickHouse/ClickHouse/pull/60494) ([Robert Schulze](https://github.com/rschu1ze)). +* Renamed server setting `dns_cache_max_size` to `dns_cache_max_entries` to reduce ambiguity. [#60500](https://github.com/ClickHouse/ClickHouse/pull/60500) ([Kirill Nikiforov](https://github.com/allmazz)). +* `SHOW INDEX | INDEXES | INDICES | KEYS` no longer sorts by the primary key columns (which was unintuitive). [#60514](https://github.com/ClickHouse/ClickHouse/pull/60514) ([Robert Schulze](https://github.com/rschu1ze)). +* Keeper improvement: abort during startup if an invalid snapshot is detected to avoid data loss. [#60537](https://github.com/ClickHouse/ClickHouse/pull/60537) ([Antonio Andelic](https://github.com/antonio2368)). +* Added MergeTree read split ranges into intersecting and non intersecting fault injection using `merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_fault_probability` setting. [#60548](https://github.com/ClickHouse/ClickHouse/pull/60548) ([Maksim Kita](https://github.com/kitaisreal)). +* The Advanced dashboard now has controls always visible on scrolling. This allows you to add a new chart without scrolling up. [#60692](https://github.com/ClickHouse/ClickHouse/pull/60692) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* String types and Enums can be used in the same context, such as: arrays, UNION queries, conditional expressions. This closes [#60726](https://github.com/ClickHouse/ClickHouse/issues/60726). [#60727](https://github.com/ClickHouse/ClickHouse/pull/60727) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update tzdata to 2024a. [#60768](https://github.com/ClickHouse/ClickHouse/pull/60768) ([Raúl Marín](https://github.com/Algunenano)). +* Support files without format extension in Filesystem database. [#60795](https://github.com/ClickHouse/ClickHouse/pull/60795) ([Kruglov Pavel](https://github.com/Avogar)). +* Keeper improvement: support `leadership_expiry_ms` in Keeper's settings. [#60806](https://github.com/ClickHouse/ClickHouse/pull/60806) ([Brokenice0415](https://github.com/Brokenice0415)). +* Always infer exponential numbers in JSON formats regardless of the setting `input_format_try_infer_exponent_floats`. Add setting `input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects` that allows to use String type for ambiguous paths instead of an exception during named Tuples inference from JSON objects. [#60808](https://github.com/ClickHouse/ClickHouse/pull/60808) ([Kruglov Pavel](https://github.com/Avogar)). +* Add support for `START TRANSACTION` syntax typically used in MySQL syntax, resolving https://github.com/ClickHouse/ClickHouse/discussions/60865. [#60886](https://github.com/ClickHouse/ClickHouse/pull/60886) ([Zach Naimon](https://github.com/ArctypeZach)). +* Add a flag for SMJ to treat null as biggest/smallest. So the behavior can be compitable with other SQL systems, like Apache Spark. [#60896](https://github.com/ClickHouse/ClickHouse/pull/60896) ([loudongfeng](https://github.com/loudongfeng)). +* Clickhouse version has been added to docker labels. Closes [#54224](https://github.com/ClickHouse/ClickHouse/issues/54224). [#60949](https://github.com/ClickHouse/ClickHouse/pull/60949) ([Nikolay Monkov](https://github.com/nikmonkov)). +* Add a setting `parallel_replicas_allow_in_with_subquery = 1` which allows subqueries for IN work with parallel replicas. [#60950](https://github.com/ClickHouse/ClickHouse/pull/60950) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* DNSResolver shuffles set of resolved IPs. [#60965](https://github.com/ClickHouse/ClickHouse/pull/60965) ([Sema Checherinda](https://github.com/CheSema)). +* Support detect output format by file exctension in `clickhouse-client` and `clickhouse-local`. [#61036](https://github.com/ClickHouse/ClickHouse/pull/61036) ([豪肥肥](https://github.com/HowePa)). +* Check memory limit update periodically. [#61049](https://github.com/ClickHouse/ClickHouse/pull/61049) ([Han Fei](https://github.com/hanfei1991)). +* Enable processors profiling (time spent/in and out bytes for sorting, aggregation, ...) by default. [#61096](https://github.com/ClickHouse/ClickHouse/pull/61096) ([Azat Khuzhin](https://github.com/azat)). +* Add the function `toUInt128OrZero`, which was missed by mistake (the mistake is related to https://github.com/ClickHouse/ClickHouse/pull/945). The compatibility aliases `FROM_UNIXTIME` and `DATE_FORMAT` (they are not ClickHouse-native and only exist for MySQL compatibility) have been made case insensitive, as expected for SQL-compatibility aliases. [#61114](https://github.com/ClickHouse/ClickHouse/pull/61114) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improvements for the access checks, allowing to revoke of unpossessed rights in case the target user doesn't have the revoking grants either. Example: ```sql GRANT SELECT ON *.* TO user1; REVOKE SELECT ON system.* FROM user1;. [#61115](https://github.com/ClickHouse/ClickHouse/pull/61115) ([pufit](https://github.com/pufit)). +* Fix an error in previeous opt: https://github.com/ClickHouse/ClickHouse/pull/59698: remove break to make sure the first filtered column has minimum size cc @jsc0218. [#61145](https://github.com/ClickHouse/ClickHouse/pull/61145) ([李扬](https://github.com/taiyang-li)). +* Fix `has()` function with `Nullable` column (fixes [#60214](https://github.com/ClickHouse/ClickHouse/issues/60214)). [#61249](https://github.com/ClickHouse/ClickHouse/pull/61249) ([Mikhail Koviazin](https://github.com/mkmkme)). +* Now it's possible to specify attribute `merge="true"` in config substitutions for subtrees ``. In case this attribute specified, clickhouse will merge subtree with existing configuration, otherwise default behavior is append new content to configuration. [#61299](https://github.com/ClickHouse/ClickHouse/pull/61299) ([alesapin](https://github.com/alesapin)). +* Add async metrics for virtual memory mappings: VMMaxMapCount & VMNumMaps. Closes [#60662](https://github.com/ClickHouse/ClickHouse/issues/60662). [#61354](https://github.com/ClickHouse/ClickHouse/pull/61354) ([Tuan Pham Anh](https://github.com/tuanpavn)). +* Use `temporary_files_codec` setting in all places where we create temporary data, for example external memory sorting and external memory GROUP BY. Before it worked only in `partial_merge` JOIN algorithm. [#61456](https://github.com/ClickHouse/ClickHouse/pull/61456) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove duplicated check `containing_part.empty()`, It's already being checked here: https://github.com/ClickHouse/ClickHouse/blob/1296dac3c7e47670872c15e3f5e58f869e0bd2f2/src/Storages/MergeTree/MergeTreeData.cpp#L6141. [#61467](https://github.com/ClickHouse/ClickHouse/pull/61467) ([William Schoeffel](https://github.com/wiledusc)). +* Add a new setting `max_parser_backtracks` which allows to limit the complexity of query parsing. [#61502](https://github.com/ClickHouse/ClickHouse/pull/61502) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support parallel reading for azure blob storage. [#61503](https://github.com/ClickHouse/ClickHouse/pull/61503) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Less contention during dynamic resize of filesystem cache. [#61524](https://github.com/ClickHouse/ClickHouse/pull/61524) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disallow sharded mode of StorageS3 queue, because it will be rewritten. [#61537](https://github.com/ClickHouse/ClickHouse/pull/61537) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed typo: from `use_leagcy_max_level` to `use_legacy_max_level`. [#61545](https://github.com/ClickHouse/ClickHouse/pull/61545) ([William Schoeffel](https://github.com/wiledusc)). +* Remove some duplicate entries in blob_storage_log. [#61622](https://github.com/ClickHouse/ClickHouse/pull/61622) ([YenchangChan](https://github.com/YenchangChan)). +* Enable `allow_experimental_analyzer` setting by default. [#61652](https://github.com/ClickHouse/ClickHouse/pull/61652) ([Dmitry Novik](https://github.com/novikd)). +* Added `current_user` function as a compatibility alias for MySQL. [#61770](https://github.com/ClickHouse/ClickHouse/pull/61770) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Use managed identity for backups IO when using Azure Blob Storage. Add a setting to prevent ClickHouse from attempting to create a non-existent container, which requires permissions at the storage account level. [#61785](https://github.com/ClickHouse/ClickHouse/pull/61785) ([Daniel Pozo Escalona](https://github.com/danipozo)). +* Enable `output_format_pretty_row_numbers` by default. It is better for usability. [#61791](https://github.com/ClickHouse/ClickHouse/pull/61791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In the previous version, some numbers in Pretty formats were not pretty enough. [#61794](https://github.com/ClickHouse/ClickHouse/pull/61794) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A long value in Pretty formats won't be cut if it is the single value in the resultset, such as in the result of the `SHOW CREATE TABLE` query. [#61795](https://github.com/ClickHouse/ClickHouse/pull/61795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Similarly to `clickhouse-local`, `clickhouse-client` will accept the `--output-format` option as a synonym to the `--format` option. This closes [#59848](https://github.com/ClickHouse/ClickHouse/issues/59848). [#61797](https://github.com/ClickHouse/ClickHouse/pull/61797) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* If stdout is a terminal and the output format is not specified, `clickhouse-client` and similar tools will use `PrettyCompact` by default, similarly to the interactive mode. `clickhouse-client` and `clickhouse-local` will handle command line arguments for input and output formats in a unified fashion. This closes [#61272](https://github.com/ClickHouse/ClickHouse/issues/61272). [#61800](https://github.com/ClickHouse/ClickHouse/pull/61800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Underscore digit groups in Pretty formats for better readability. This is controlled by a new setting, `output_format_pretty_highlight_digit_groups`. [#61802](https://github.com/ClickHouse/ClickHouse/pull/61802) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add ability to override initial INSERT SETTINGS via SYSTEM FLUSH DISTRIBUTED. [#61832](https://github.com/ClickHouse/ClickHouse/pull/61832) ([Azat Khuzhin](https://github.com/azat)). +* Fixed grammar from "a" to "the" in the warning message. There is only one Atomic engine, so it should be "to the new Atomic engine" instead of "to a new Atomic engine". [#61952](https://github.com/ClickHouse/ClickHouse/pull/61952) ([shabroo](https://github.com/shabroo)). + +#### Build/Testing/Packaging Improvement +* Update sccache to the latest version; significantly reduce images size by reshaking the dependency trees; use the latest working odbc driver. [#59953](https://github.com/ClickHouse/ClickHouse/pull/59953) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update python related style checkers. Continue the [#50174](https://github.com/ClickHouse/ClickHouse/issues/50174). [#60408](https://github.com/ClickHouse/ClickHouse/pull/60408) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Upgrade `prqlc` to 0.11.3. [#60616](https://github.com/ClickHouse/ClickHouse/pull/60616) ([Maximilian Roos](https://github.com/max-sixty)). +* Attach gdb to running fuzzer process. [#60654](https://github.com/ClickHouse/ClickHouse/pull/60654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use explicit template instantiation more aggressively. Get rid of templates in favor of overloaded functions in some places. [#60730](https://github.com/ClickHouse/ClickHouse/pull/60730) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* The real-time query profiler now works on AArch64. In previous versions, it worked only when a program didn't spend time inside a syscall. [#60807](https://github.com/ClickHouse/ClickHouse/pull/60807) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ... Too big translation unit in `Aggregator`. [#61211](https://github.com/ClickHouse/ClickHouse/pull/61211) ([lgbo](https://github.com/lgbo-ustc)). +* Fixed flakiness of 01603_insert_select_too_many_parts test. Closes [#61158](https://github.com/ClickHouse/ClickHouse/issues/61158). [#61259](https://github.com/ClickHouse/ClickHouse/pull/61259) ([Ilya Yatsishin](https://github.com/qoega)). +* Now it possible to use `chassert(expression, comment)` in the codebase. [#61263](https://github.com/ClickHouse/ClickHouse/pull/61263) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Teach the fuzzer to use other numeric types. [#61317](https://github.com/ClickHouse/ClickHouse/pull/61317) ([Raúl Marín](https://github.com/Algunenano)). +* Increase memory limit for coverage builds. [#61405](https://github.com/ClickHouse/ClickHouse/pull/61405) ([Raúl Marín](https://github.com/Algunenano)). +* Add generic query text fuzzer in `clickhouse-local`. [#61508](https://github.com/ClickHouse/ClickHouse/pull/61508) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix function execution over const and LowCardinality with GROUP BY const for analyzer [#59986](https://github.com/ClickHouse/ClickHouse/pull/59986) ([Azat Khuzhin](https://github.com/azat)). +* Fix finished_mutations_to_keep=0 for MergeTree (as docs says 0 is to keep everything) [#60031](https://github.com/ClickHouse/ClickHouse/pull/60031) ([Azat Khuzhin](https://github.com/azat)). +* PartsSplitter invalid ranges for the same part [#60041](https://github.com/ClickHouse/ClickHouse/pull/60041) ([Maksim Kita](https://github.com/kitaisreal)). +* Azure Blob Storage : Fix issues endpoint and prefix [#60251](https://github.com/ClickHouse/ClickHouse/pull/60251) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* fix LRUResource Cache bug (Hive cache) [#60262](https://github.com/ClickHouse/ClickHouse/pull/60262) ([shanfengp](https://github.com/Aed-p)). +* Force reanalysis if parallel replicas changed [#60362](https://github.com/ClickHouse/ClickHouse/pull/60362) ([Raúl Marín](https://github.com/Algunenano)). +* Fix usage of plain metadata type with new disks configuration option [#60396](https://github.com/ClickHouse/ClickHouse/pull/60396) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Try to fix logical error 'Cannot capture column because it has incompatible type' in mapContainsKeyLike [#60451](https://github.com/ClickHouse/ClickHouse/pull/60451) ([Kruglov Pavel](https://github.com/Avogar)). +* Try to avoid calculation of scalar subqueries for CREATE TABLE. [#60464](https://github.com/ClickHouse/ClickHouse/pull/60464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix deadlock in parallel parsing when lots of rows are skipped due to errors [#60516](https://github.com/ClickHouse/ClickHouse/pull/60516) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix_max_query_size_for_kql_compound_operator: [#60534](https://github.com/ClickHouse/ClickHouse/pull/60534) ([Yong Wang](https://github.com/kashwy)). +* Keeper fix: add timeouts when waiting for commit logs [#60544](https://github.com/ClickHouse/ClickHouse/pull/60544) ([Antonio Andelic](https://github.com/antonio2368)). +* Reduce the number of read rows from `system.numbers` [#60546](https://github.com/ClickHouse/ClickHouse/pull/60546) ([JackyWoo](https://github.com/JackyWoo)). +* Don't output number tips for date types [#60577](https://github.com/ClickHouse/ClickHouse/pull/60577) ([Raúl Marín](https://github.com/Algunenano)). +* Fix reading from MergeTree with non-deterministic functions in filter [#60586](https://github.com/ClickHouse/ClickHouse/pull/60586) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix logical error on bad compatibility setting value type [#60596](https://github.com/ClickHouse/ClickHouse/pull/60596) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix inconsistent aggregate function states in mixed x86-64 / ARM clusters [#60610](https://github.com/ClickHouse/ClickHouse/pull/60610) ([Harry Lee](https://github.com/HarryLeeIBM)). +* fix(prql): Robust panic handler [#60615](https://github.com/ClickHouse/ClickHouse/pull/60615) ([Maximilian Roos](https://github.com/max-sixty)). +* Fix `intDiv` for decimal and date arguments [#60672](https://github.com/ClickHouse/ClickHouse/pull/60672) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Fix: expand CTE in alter modify query [#60682](https://github.com/ClickHouse/ClickHouse/pull/60682) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix system.parts for non-Atomic/Ordinary database engine (i.e. Memory) [#60689](https://github.com/ClickHouse/ClickHouse/pull/60689) ([Azat Khuzhin](https://github.com/azat)). +* Fix "Invalid storage definition in metadata file" for parameterized views [#60708](https://github.com/ClickHouse/ClickHouse/pull/60708) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer overflow in CompressionCodecMultiple [#60731](https://github.com/ClickHouse/ClickHouse/pull/60731) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove nonsense from SQL/JSON [#60738](https://github.com/ClickHouse/ClickHouse/pull/60738) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove wrong sanitize checking in aggregate function quantileGK [#60740](https://github.com/ClickHouse/ClickHouse/pull/60740) ([李扬](https://github.com/taiyang-li)). +* Fix insert-select + insert_deduplication_token bug by setting streams to 1 [#60745](https://github.com/ClickHouse/ClickHouse/pull/60745) ([Jordi Villar](https://github.com/jrdi)). +* Prevent setting custom metadata headers on unsupported multipart upload operations [#60748](https://github.com/ClickHouse/ClickHouse/pull/60748) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)). +* Fix toStartOfInterval [#60763](https://github.com/ClickHouse/ClickHouse/pull/60763) ([Andrey Zvonov](https://github.com/zvonand)). +* Fix crash in arrayEnumerateRanked [#60764](https://github.com/ClickHouse/ClickHouse/pull/60764) ([Raúl Marín](https://github.com/Algunenano)). +* Fix crash when using input() in INSERT SELECT JOIN [#60765](https://github.com/ClickHouse/ClickHouse/pull/60765) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash with different allow_experimental_analyzer value in subqueries [#60770](https://github.com/ClickHouse/ClickHouse/pull/60770) ([Dmitry Novik](https://github.com/novikd)). +* Remove recursion when reading from S3 [#60849](https://github.com/ClickHouse/ClickHouse/pull/60849) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix possible stuck on error in HashedDictionaryParallelLoader [#60926](https://github.com/ClickHouse/ClickHouse/pull/60926) ([vdimir](https://github.com/vdimir)). +* Fix async RESTORE with Replicated database [#60934](https://github.com/ClickHouse/ClickHouse/pull/60934) ([Antonio Andelic](https://github.com/antonio2368)). +* fix csv format not support tuple [#60994](https://github.com/ClickHouse/ClickHouse/pull/60994) ([shuai.xu](https://github.com/shuai-xu)). +* Fix deadlock in async inserts to `Log` tables via native protocol [#61055](https://github.com/ClickHouse/ClickHouse/pull/61055) ([Anton Popov](https://github.com/CurtizJ)). +* Fix lazy execution of default argument in dictGetOrDefault for RangeHashedDictionary [#61196](https://github.com/ClickHouse/ClickHouse/pull/61196) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix multiple bugs in groupArraySorted [#61203](https://github.com/ClickHouse/ClickHouse/pull/61203) ([Raúl Marín](https://github.com/Algunenano)). +* Fix Keeper reconfig for standalone binary [#61233](https://github.com/ClickHouse/ClickHouse/pull/61233) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix usage of session_token in S3 engine [#61234](https://github.com/ClickHouse/ClickHouse/pull/61234) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix possible incorrect result of aggregate function `uniqExact` [#61257](https://github.com/ClickHouse/ClickHouse/pull/61257) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bugs in show database [#61269](https://github.com/ClickHouse/ClickHouse/pull/61269) ([Raúl Marín](https://github.com/Algunenano)). +* Fix logical error in RabbitMQ storage with MATERIALIZED columns [#61320](https://github.com/ClickHouse/ClickHouse/pull/61320) ([vdimir](https://github.com/vdimir)). +* Fix CREATE OR REPLACE DICTIONARY [#61356](https://github.com/ClickHouse/ClickHouse/pull/61356) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix crash in ObjectJson parsing array with nulls [#61364](https://github.com/ClickHouse/ClickHouse/pull/61364) ([vdimir](https://github.com/vdimir)). +* Fix ATTACH query with external ON CLUSTER [#61365](https://github.com/ClickHouse/ClickHouse/pull/61365) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix consecutive keys optimization for nullable keys [#61393](https://github.com/ClickHouse/ClickHouse/pull/61393) ([Anton Popov](https://github.com/CurtizJ)). +* fix issue of actions dag split [#61458](https://github.com/ClickHouse/ClickHouse/pull/61458) ([Raúl Marín](https://github.com/Algunenano)). +* Fix finishing a failed RESTORE [#61466](https://github.com/ClickHouse/ClickHouse/pull/61466) ([Vitaly Baranov](https://github.com/vitlibar)). +* Disable async_insert_use_adaptive_busy_timeout correctly with compatibility settings [#61468](https://github.com/ClickHouse/ClickHouse/pull/61468) ([Raúl Marín](https://github.com/Algunenano)). +* Allow queuing in restore pool [#61475](https://github.com/ClickHouse/ClickHouse/pull/61475) ([Nikita Taranov](https://github.com/nickitat)). +* Fix bug when reading system.parts using UUID (issue 61220). [#61479](https://github.com/ClickHouse/ClickHouse/pull/61479) ([Dan Wu](https://github.com/wudanzy)). +* Fix ALTER QUERY MODIFY SQL SECURITY [#61480](https://github.com/ClickHouse/ClickHouse/pull/61480) ([pufit](https://github.com/pufit)). +* Fix crash in window view [#61526](https://github.com/ClickHouse/ClickHouse/pull/61526) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `repeat` with non native integers [#61527](https://github.com/ClickHouse/ClickHouse/pull/61527) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix client `-s` argument [#61530](https://github.com/ClickHouse/ClickHouse/pull/61530) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Reset part level upon attach from disk on MergeTree [#61536](https://github.com/ClickHouse/ClickHouse/pull/61536) ([Arthur Passos](https://github.com/arthurpassos)). +* Fix crash in arrayPartialReverseSort [#61539](https://github.com/ClickHouse/ClickHouse/pull/61539) ([Raúl Marín](https://github.com/Algunenano)). +* Fix string search with const position [#61547](https://github.com/ClickHouse/ClickHouse/pull/61547) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix addDays cause an error when used datetime64 [#61561](https://github.com/ClickHouse/ClickHouse/pull/61561) ([Shuai li](https://github.com/loneylee)). +* disallow LowCardinality input type for JSONExtract [#61617](https://github.com/ClickHouse/ClickHouse/pull/61617) ([Julia Kartseva](https://github.com/jkartseva)). +* Fix `system.part_log` for async insert with deduplication [#61620](https://github.com/ClickHouse/ClickHouse/pull/61620) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix Non-ready set for system.parts. [#61666](https://github.com/ClickHouse/ClickHouse/pull/61666) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Don't allow the same expression in ORDER BY with and without WITH FILL [#61667](https://github.com/ClickHouse/ClickHouse/pull/61667) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix actual_part_name for REPLACE_RANGE (`Entry actual part isn't empty yet`) [#61675](https://github.com/ClickHouse/ClickHouse/pull/61675) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix columns after executing MODIFY QUERY for a materialized view with internal table [#61734](https://github.com/ClickHouse/ClickHouse/pull/61734) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix crash in `multiSearchAllPositionsCaseInsensitiveUTF8` for incorrect UTF-8 [#61749](https://github.com/ClickHouse/ClickHouse/pull/61749) ([pufit](https://github.com/pufit)). +* Fix RANGE frame is not supported for Nullable columns. [#61766](https://github.com/ClickHouse/ClickHouse/pull/61766) ([YuanLiu](https://github.com/ditgittube)). +* Revert "Revert "Fix bug when reading system.parts using UUID (issue 61220)."" [#61779](https://github.com/ClickHouse/ClickHouse/pull/61779) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). + +#### CI Fix or Improvement (changelog entry is not required) + +* Decoupled changes from [#60408](https://github.com/ClickHouse/ClickHouse/issues/60408). [#60553](https://github.com/ClickHouse/ClickHouse/pull/60553) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Eliminates the need to provide input args to docker server jobs to clean yml files. [#60602](https://github.com/ClickHouse/ClickHouse/pull/60602) ([Max K.](https://github.com/maxknv)). +* Debug and fix markreleaseready. [#60611](https://github.com/ClickHouse/ClickHouse/pull/60611) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix build_report job so that it's defined by ci_config only (not yml file). [#60613](https://github.com/ClickHouse/ClickHouse/pull/60613) ([Max K.](https://github.com/maxknv)). +* Do not await ci pending jobs on release branches decrease wait timeout to fit into gh job timeout. [#60652](https://github.com/ClickHouse/ClickHouse/pull/60652) ([Max K.](https://github.com/maxknv)). +* Set limited number of builds for "special build check" report in backports. [#60850](https://github.com/ClickHouse/ClickHouse/pull/60850) ([Max K.](https://github.com/maxknv)). +* ... [#60935](https://github.com/ClickHouse/ClickHouse/pull/60935) ([Max K.](https://github.com/maxknv)). +* ... [#60947](https://github.com/ClickHouse/ClickHouse/pull/60947) ([Max K.](https://github.com/maxknv)). +* ... [#60952](https://github.com/ClickHouse/ClickHouse/pull/60952) ([Max K.](https://github.com/maxknv)). +* ... [#60958](https://github.com/ClickHouse/ClickHouse/pull/60958) ([Max K.](https://github.com/maxknv)). +* ... [#61022](https://github.com/ClickHouse/ClickHouse/pull/61022) ([Max K.](https://github.com/maxknv)). +* Just a preparation for the merge queue support. [#61099](https://github.com/ClickHouse/ClickHouse/pull/61099) ([Max K.](https://github.com/maxknv)). +* ... [#61133](https://github.com/ClickHouse/ClickHouse/pull/61133) ([Max K.](https://github.com/maxknv)). +* In PRs: - run typos, aspell check - always - run pylint, mypy - only if py file(s) changed in PRs - run basic source files style check - only if not all changes in py files. [#61148](https://github.com/ClickHouse/ClickHouse/pull/61148) ([Max K.](https://github.com/maxknv)). +* ... [#61172](https://github.com/ClickHouse/ClickHouse/pull/61172) ([Max K.](https://github.com/maxknv)). +* ... [#61183](https://github.com/ClickHouse/ClickHouse/pull/61183) ([Han Fei](https://github.com/hanfei1991)). +* ... [#61185](https://github.com/ClickHouse/ClickHouse/pull/61185) ([Max K.](https://github.com/maxknv)). +* TBD. [#61197](https://github.com/ClickHouse/ClickHouse/pull/61197) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* ... [#61214](https://github.com/ClickHouse/ClickHouse/pull/61214) ([Max K.](https://github.com/maxknv)). +* ... [#61441](https://github.com/ClickHouse/ClickHouse/pull/61441) ([Max K.](https://github.com/maxknv)). +* ![Screenshot_20240323_025055](https://github.com/ClickHouse/ClickHouse/assets/18581488/ccaab212-a1d3-4dfb-8d56-b1991760b6bf). [#61801](https://github.com/ClickHouse/ClickHouse/pull/61801) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ... [#61877](https://github.com/ClickHouse/ClickHouse/pull/61877) ([Max K.](https://github.com/maxknv)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Revert "Use `MergeTree` as a default table engine""'. [#60524](https://github.com/ClickHouse/ClickHouse/pull/60524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Revert "Support resource request canceling""'. [#60558](https://github.com/ClickHouse/ClickHouse/pull/60558) ([Sergei Trifonov](https://github.com/serxa)). +* NO CL ENTRY: 'Revert "Add `toMillisecond` function"'. [#60644](https://github.com/ClickHouse/ClickHouse/pull/60644) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Synchronize parsers"'. [#60759](https://github.com/ClickHouse/ClickHouse/pull/60759) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Fix wacky primary key sorting in `SHOW INDEX`"'. [#60898](https://github.com/ClickHouse/ClickHouse/pull/60898) ([Antonio Andelic](https://github.com/antonio2368)). +* NO CL ENTRY: 'Revert "CI: make style check faster"'. [#61142](https://github.com/ClickHouse/ClickHouse/pull/61142) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Don't allow to set max_parallel_replicas to 0 as it doesn't make sense"'. [#61200](https://github.com/ClickHouse/ClickHouse/pull/61200) ([Kruglov Pavel](https://github.com/Avogar)). +* NO CL ENTRY: 'Revert "Fix usage of session_token in S3 engine"'. [#61359](https://github.com/ClickHouse/ClickHouse/pull/61359) ([Antonio Andelic](https://github.com/antonio2368)). +* NO CL ENTRY: 'Revert "Revert "Fix usage of session_token in S3 engine""'. [#61362](https://github.com/ClickHouse/ClickHouse/pull/61362) ([Kruglov Pavel](https://github.com/Avogar)). +* NO CL ENTRY: 'Reorder hidden and shown checks in comment, change url of Mergeable check'. [#61373](https://github.com/ClickHouse/ClickHouse/pull/61373) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* NO CL ENTRY: 'Remove unnecessary layers from clickhouse/cctools'. [#61374](https://github.com/ClickHouse/ClickHouse/pull/61374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* NO CL ENTRY: 'Revert "Updated format settings references in the docs (datetime.md)"'. [#61435](https://github.com/ClickHouse/ClickHouse/pull/61435) ([Kruglov Pavel](https://github.com/Avogar)). +* NO CL ENTRY: 'Revert "CI: ARM integration tests: disable tests with HDFS "'. [#61449](https://github.com/ClickHouse/ClickHouse/pull/61449) ([Max K.](https://github.com/maxknv)). +* NO CL ENTRY: 'Revert "Analyzer: Fix virtual columns in StorageMerge"'. [#61518](https://github.com/ClickHouse/ClickHouse/pull/61518) ([Antonio Andelic](https://github.com/antonio2368)). +* NO CL ENTRY: 'Revert "Revert "Analyzer: Fix virtual columns in StorageMerge""'. [#61528](https://github.com/ClickHouse/ClickHouse/pull/61528) ([Dmitry Novik](https://github.com/novikd)). +* NO CL ENTRY: 'Improve build_download_helper'. [#61592](https://github.com/ClickHouse/ClickHouse/pull/61592) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* NO CL ENTRY: 'Revert "Un-flake `test_undrop_query`"'. [#61668](https://github.com/ClickHouse/ClickHouse/pull/61668) ([Robert Schulze](https://github.com/rschu1ze)). +* NO CL ENTRY: 'Fix flaky tests (stateless, integration)'. [#61816](https://github.com/ClickHouse/ClickHouse/pull/61816) ([Nikita Fomichev](https://github.com/fm4v)). +* NO CL ENTRY: 'Better usability of "expect" tests: less trouble with running directly'. [#61818](https://github.com/ClickHouse/ClickHouse/pull/61818) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Fix flaky `02122_parallel_formatting_Template`"'. [#61868](https://github.com/ClickHouse/ClickHouse/pull/61868) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Add --now option to enable and start the service" #job_Install_packages_amd64'. [#61878](https://github.com/ClickHouse/ClickHouse/pull/61878) ([Max K.](https://github.com/maxknv)). +* NO CL ENTRY: 'Revert "disallow LowCardinality input type for JSONExtract"'. [#61960](https://github.com/ClickHouse/ClickHouse/pull/61960) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve query performance in case of very small blocks [#58879](https://github.com/ClickHouse/ClickHouse/pull/58879) ([Azat Khuzhin](https://github.com/azat)). +* Analyzer: fixes for JOIN columns resolution [#59007](https://github.com/ClickHouse/ClickHouse/pull/59007) ([vdimir](https://github.com/vdimir)). +* Fix race on `Context::async_insert_queue` [#59082](https://github.com/ClickHouse/ClickHouse/pull/59082) ([Alexander Tokmakov](https://github.com/tavplubix)). +* CI: support batch specification in commit message [#59738](https://github.com/ClickHouse/ClickHouse/pull/59738) ([Max K.](https://github.com/maxknv)). +* Update storing-data.md [#60024](https://github.com/ClickHouse/ClickHouse/pull/60024) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make max_insert_delayed_streams_for_parallel_write actually work [#60079](https://github.com/ClickHouse/ClickHouse/pull/60079) ([alesapin](https://github.com/alesapin)). +* Analyzer: support join using column from select list [#60182](https://github.com/ClickHouse/ClickHouse/pull/60182) ([vdimir](https://github.com/vdimir)). +* test for [#60223](https://github.com/ClickHouse/ClickHouse/issues/60223) [#60258](https://github.com/ClickHouse/ClickHouse/pull/60258) ([Denny Crane](https://github.com/den-crane)). +* Analyzer: Refactor execution name for ConstantNode [#60313](https://github.com/ClickHouse/ClickHouse/pull/60313) ([Dmitry Novik](https://github.com/novikd)). +* Fix database iterator waiting code [#60314](https://github.com/ClickHouse/ClickHouse/pull/60314) ([Sergei Trifonov](https://github.com/serxa)). +* QueryCache: Don't acquire the query count mutex if not necessary [#60348](https://github.com/ClickHouse/ClickHouse/pull/60348) ([zhongyuankai](https://github.com/zhongyuankai)). +* Fix bugfix check (due to unknown commit_logs_cache_size_threshold) [#60375](https://github.com/ClickHouse/ClickHouse/pull/60375) ([Azat Khuzhin](https://github.com/azat)). +* Enable testing with `io_uring` back [#60383](https://github.com/ClickHouse/ClickHouse/pull/60383) ([Nikita Taranov](https://github.com/nickitat)). +* Analyzer - improve hiding secret arguments. [#60386](https://github.com/ClickHouse/ClickHouse/pull/60386) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* CI: make workflow yml abstract [#60421](https://github.com/ClickHouse/ClickHouse/pull/60421) ([Max K.](https://github.com/maxknv)). +* Improve test test_reload_clusters_config [#60426](https://github.com/ClickHouse/ClickHouse/pull/60426) ([Kruglov Pavel](https://github.com/Avogar)). +* Revert "Revert "Merge pull request [#56864](https://github.com/ClickHouse/ClickHouse/issues/56864) from ClickHouse/broken-projections-better-handling"" [#60452](https://github.com/ClickHouse/ClickHouse/pull/60452) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Do not check to and from files existence in metadata_storage because it does not see uncommitted changes [#60462](https://github.com/ClickHouse/ClickHouse/pull/60462) ([Alexander Gololobov](https://github.com/davenger)). +* Fix option ambiguous in `clickhouse-local` [#60475](https://github.com/ClickHouse/ClickHouse/pull/60475) ([豪肥肥](https://github.com/HowePa)). +* Fix: test_parallel_replicas_custom_key_load_balancing [#60485](https://github.com/ClickHouse/ClickHouse/pull/60485) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix: progress bar for *Cluster table functions [#60491](https://github.com/ClickHouse/ClickHouse/pull/60491) ([Igor Nikonov](https://github.com/devcrafter)). +* Analyzer: Support different ObjectJSON on shards [#60497](https://github.com/ClickHouse/ClickHouse/pull/60497) ([Dmitry Novik](https://github.com/novikd)). +* Cancel PipelineExecutor properly in case of exception in spawnThreads [#60499](https://github.com/ClickHouse/ClickHouse/pull/60499) ([Kruglov Pavel](https://github.com/Avogar)). +* Refactor StorageSystemOneBlock [#60510](https://github.com/ClickHouse/ClickHouse/pull/60510) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Simple cleanup while fixing progress bar [#60513](https://github.com/ClickHouse/ClickHouse/pull/60513) ([Igor Nikonov](https://github.com/devcrafter)). +* PullingAsyncPipelineExecutor cleanup [#60515](https://github.com/ClickHouse/ClickHouse/pull/60515) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix bad error message [#60518](https://github.com/ClickHouse/ClickHouse/pull/60518) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Synchronize Access [#60519](https://github.com/ClickHouse/ClickHouse/pull/60519) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Synchronize metrics and Keeper [#60520](https://github.com/ClickHouse/ClickHouse/pull/60520) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enforce clang-tidy in `programs/` and `utils/` headers [#60521](https://github.com/ClickHouse/ClickHouse/pull/60521) ([Robert Schulze](https://github.com/rschu1ze)). +* Synchronize parsers [#60522](https://github.com/ClickHouse/ClickHouse/pull/60522) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a bunch of clang-tidy warnings in headers [#60523](https://github.com/ClickHouse/ClickHouse/pull/60523) ([Robert Schulze](https://github.com/rschu1ze)). +* General sanity in function `seriesOutliersDetectTukey` [#60535](https://github.com/ClickHouse/ClickHouse/pull/60535) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Chinese document for max_query_size, max_parser_depth and optimize_functions_to_subcolumns [#60541](https://github.com/ClickHouse/ClickHouse/pull/60541) ([Alex Cheng](https://github.com/Alex-Cheng)). +* Userspace page cache again [#60552](https://github.com/ClickHouse/ClickHouse/pull/60552) ([Michael Kolupaev](https://github.com/al13n321)). +* Traverse shadow directory for system.remote_data_paths [#60585](https://github.com/ClickHouse/ClickHouse/pull/60585) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Add test for [#58906](https://github.com/ClickHouse/ClickHouse/issues/58906) [#60597](https://github.com/ClickHouse/ClickHouse/pull/60597) ([Raúl Marín](https://github.com/Algunenano)). +* Use python zipfile to have x-platform idempotent lambda packages [#60603](https://github.com/ClickHouse/ClickHouse/pull/60603) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* tests: suppress data-race in librdkafka statistics code [#60604](https://github.com/ClickHouse/ClickHouse/pull/60604) ([Azat Khuzhin](https://github.com/azat)). +* Update version after release [#60605](https://github.com/ClickHouse/ClickHouse/pull/60605) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v24.2.1.2248-stable [#60607](https://github.com/ClickHouse/ClickHouse/pull/60607) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Addition to changelog [#60609](https://github.com/ClickHouse/ClickHouse/pull/60609) ([Anton Popov](https://github.com/CurtizJ)). +* internal: Refine rust prql code [#60617](https://github.com/ClickHouse/ClickHouse/pull/60617) ([Maximilian Roos](https://github.com/max-sixty)). +* fix(rust): Fix skim's panic handler [#60621](https://github.com/ClickHouse/ClickHouse/pull/60621) ([Maximilian Roos](https://github.com/max-sixty)). +* Resubmit "Analyzer: compute ALIAS columns right after reading" [#60641](https://github.com/ClickHouse/ClickHouse/pull/60641) ([vdimir](https://github.com/vdimir)). +* Analyzer: Fix bug with join_use_nulls and PREWHERE [#60655](https://github.com/ClickHouse/ClickHouse/pull/60655) ([vdimir](https://github.com/vdimir)). +* Add test for [#59891](https://github.com/ClickHouse/ClickHouse/issues/59891) [#60657](https://github.com/ClickHouse/ClickHouse/pull/60657) ([Raúl Marín](https://github.com/Algunenano)). +* Fix missed entries in system.part_log in case of fetch preferred over merges/mutations [#60659](https://github.com/ClickHouse/ClickHouse/pull/60659) ([Azat Khuzhin](https://github.com/azat)). +* Always apply first minmax index among available skip indices [#60675](https://github.com/ClickHouse/ClickHouse/pull/60675) ([Igor Nikonov](https://github.com/devcrafter)). +* Remove bad test `02152_http_external_tables_memory_tracking` [#60690](https://github.com/ClickHouse/ClickHouse/pull/60690) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix questionable behavior in the `parseDateTimeBestEffort` function. [#60691](https://github.com/ClickHouse/ClickHouse/pull/60691) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky checks [#60694](https://github.com/ClickHouse/ClickHouse/pull/60694) ([Azat Khuzhin](https://github.com/azat)). +* Resubmit http_external_tables_memory_tracking test [#60695](https://github.com/ClickHouse/ClickHouse/pull/60695) ([Azat Khuzhin](https://github.com/azat)). +* Fix bugfix and upgrade checks (due to "Unknown handler type 'redirect'" error) [#60696](https://github.com/ClickHouse/ClickHouse/pull/60696) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_grant_and_revoke/test.py::test_grant_all_on_table (after syncing with cloud) [#60699](https://github.com/ClickHouse/ClickHouse/pull/60699) ([Azat Khuzhin](https://github.com/azat)). +* Remove unit test for ColumnObject [#60709](https://github.com/ClickHouse/ClickHouse/pull/60709) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve unit tests [#60710](https://github.com/ClickHouse/ClickHouse/pull/60710) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix scheduler fairness test [#60712](https://github.com/ClickHouse/ClickHouse/pull/60712) ([Sergei Trifonov](https://github.com/serxa)). +* Do not retry queries if container is down in integration tests (resubmit) [#60714](https://github.com/ClickHouse/ClickHouse/pull/60714) ([Azat Khuzhin](https://github.com/azat)). +* Mark one setting as obsolete [#60715](https://github.com/ClickHouse/ClickHouse/pull/60715) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a test with Analyzer [#60723](https://github.com/ClickHouse/ClickHouse/pull/60723) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Two tests are fixed with Analyzer [#60724](https://github.com/ClickHouse/ClickHouse/pull/60724) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old code [#60728](https://github.com/ClickHouse/ClickHouse/pull/60728) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove more code from LIVE VIEW [#60729](https://github.com/ClickHouse/ClickHouse/pull/60729) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `test_keeper_back_to_back/test.py::test_concurrent_watches` [#60749](https://github.com/ClickHouse/ClickHouse/pull/60749) ([Antonio Andelic](https://github.com/antonio2368)). +* Catch exceptions on finalize in `InterserverIOHTTPHandler` [#60769](https://github.com/ClickHouse/ClickHouse/pull/60769) ([Antonio Andelic](https://github.com/antonio2368)). +* Reduce flakiness of 02932_refreshable_materialized_views [#60771](https://github.com/ClickHouse/ClickHouse/pull/60771) ([Michael Kolupaev](https://github.com/al13n321)). +* Use 64-bit capabilities if available [#60775](https://github.com/ClickHouse/ClickHouse/pull/60775) ([Azat Khuzhin](https://github.com/azat)). +* Include multiline logs in fuzzer fatal.log report [#60796](https://github.com/ClickHouse/ClickHouse/pull/60796) ([Raúl Marín](https://github.com/Algunenano)). +* Add missing clone calls related to compression [#60810](https://github.com/ClickHouse/ClickHouse/pull/60810) ([Raúl Marín](https://github.com/Algunenano)). +* New private runners [#60811](https://github.com/ClickHouse/ClickHouse/pull/60811) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Move userspace page cache settings to the correct section of SettingsChangeHistory.h [#60812](https://github.com/ClickHouse/ClickHouse/pull/60812) ([Michael Kolupaev](https://github.com/al13n321)). +* Update version_date.tsv and changelogs after v23.8.10.43-lts [#60851](https://github.com/ClickHouse/ClickHouse/pull/60851) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Fix fuzzer report [#60853](https://github.com/ClickHouse/ClickHouse/pull/60853) ([Raúl Marín](https://github.com/Algunenano)). +* Update version_date.tsv and changelogs after v23.3.20.27-lts [#60857](https://github.com/ClickHouse/ClickHouse/pull/60857) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Refactor OptimizeDateOrDateTimeConverterWithPreimageVisitor [#60875](https://github.com/ClickHouse/ClickHouse/pull/60875) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* Fix race in PageCache [#60878](https://github.com/ClickHouse/ClickHouse/pull/60878) ([Michael Kolupaev](https://github.com/al13n321)). +* Small changes in async inserts code [#60885](https://github.com/ClickHouse/ClickHouse/pull/60885) ([Nikita Taranov](https://github.com/nickitat)). +* Remove useless verbose logging from AWS library [#60921](https://github.com/ClickHouse/ClickHouse/pull/60921) ([alesapin](https://github.com/alesapin)). +* Throw on query timeout in ZooKeeperRetries [#60922](https://github.com/ClickHouse/ClickHouse/pull/60922) ([Antonio Andelic](https://github.com/antonio2368)). +* Bring clickhouse-test changes from private [#60924](https://github.com/ClickHouse/ClickHouse/pull/60924) ([Raúl Marín](https://github.com/Algunenano)). +* Add debug info to exceptions in `IMergeTreeDataPart::checkConsistency()` [#60981](https://github.com/ClickHouse/ClickHouse/pull/60981) ([Nikita Taranov](https://github.com/nickitat)). +* Fix a typo [#60987](https://github.com/ClickHouse/ClickHouse/pull/60987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace some header includes with forward declarations [#61003](https://github.com/ClickHouse/ClickHouse/pull/61003) ([Amos Bird](https://github.com/amosbird)). +* Speed up cctools building [#61011](https://github.com/ClickHouse/ClickHouse/pull/61011) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix ASTRenameQuery::clone [#61013](https://github.com/ClickHouse/ClickHouse/pull/61013) ([vdimir](https://github.com/vdimir)). +* Update README.md [#61021](https://github.com/ClickHouse/ClickHouse/pull/61021) ([Tyler Hannan](https://github.com/tylerhannan)). +* Fix TableFunctionExecutable::skipAnalysisForArguments [#61037](https://github.com/ClickHouse/ClickHouse/pull/61037) ([Dmitry Novik](https://github.com/novikd)). +* Fix: parallel replicas with PREWHERE (ubsan) [#61052](https://github.com/ClickHouse/ClickHouse/pull/61052) ([Igor Nikonov](https://github.com/devcrafter)). +* Fast fix tests. [#61056](https://github.com/ClickHouse/ClickHouse/pull/61056) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `test_placement_info` [#61057](https://github.com/ClickHouse/ClickHouse/pull/61057) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Fix: parallel replicas with CTEs, crash in EXPLAIN SYNTAX with analyzer [#61059](https://github.com/ClickHouse/ClickHouse/pull/61059) ([Igor Nikonov](https://github.com/devcrafter)). +* Debug fuzzer failures [#61062](https://github.com/ClickHouse/ClickHouse/pull/61062) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add regression tests for fixed issues [#61076](https://github.com/ClickHouse/ClickHouse/pull/61076) ([Antonio Andelic](https://github.com/antonio2368)). +* Analyzer: Fix 01244_optimize_distributed_group_by_sharding_key [#61089](https://github.com/ClickHouse/ClickHouse/pull/61089) ([Dmitry Novik](https://github.com/novikd)). +* Use global scalars cache with analyzer [#61104](https://github.com/ClickHouse/ClickHouse/pull/61104) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix removing is_active node after re-creation [#61105](https://github.com/ClickHouse/ClickHouse/pull/61105) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update 02962_system_sync_replica_lightweight_from_modifier.sh [#61110](https://github.com/ClickHouse/ClickHouse/pull/61110) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Simplify bridges [#61118](https://github.com/ClickHouse/ClickHouse/pull/61118) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update cppkafka to v0.4.1 [#61119](https://github.com/ClickHouse/ClickHouse/pull/61119) ([Ilya Golshtein](https://github.com/ilejn)). +* CI: add wf class in ci_config [#61122](https://github.com/ClickHouse/ClickHouse/pull/61122) ([Max K.](https://github.com/maxknv)). +* QueryFuzzer: replace element randomly when AST part buffer is full [#61124](https://github.com/ClickHouse/ClickHouse/pull/61124) ([Tomer Shafir](https://github.com/tomershafir)). +* CI: make style check fast [#61125](https://github.com/ClickHouse/ClickHouse/pull/61125) ([Max K.](https://github.com/maxknv)). +* Better gitignore [#61128](https://github.com/ClickHouse/ClickHouse/pull/61128) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix something strange [#61129](https://github.com/ClickHouse/ClickHouse/pull/61129) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update check-large-objects.sh to be language neutral [#61130](https://github.com/ClickHouse/ClickHouse/pull/61130) ([Dan Wu](https://github.com/wudanzy)). +* Throw memory limit exceptions to avoid OOM in some places [#61132](https://github.com/ClickHouse/ClickHouse/pull/61132) ([alesapin](https://github.com/alesapin)). +* Fix test_distributed_directory_monitor_split_batch_on_failure flakienss [#61136](https://github.com/ClickHouse/ClickHouse/pull/61136) ([Azat Khuzhin](https://github.com/azat)). +* Fix llvm symbolizer on CI [#61147](https://github.com/ClickHouse/ClickHouse/pull/61147) ([Azat Khuzhin](https://github.com/azat)). +* Some clang-tidy fixes [#61150](https://github.com/ClickHouse/ClickHouse/pull/61150) ([Robert Schulze](https://github.com/rschu1ze)). +* Revive "Less contention in the cache, part 2" [#61152](https://github.com/ClickHouse/ClickHouse/pull/61152) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Enable black back [#61159](https://github.com/ClickHouse/ClickHouse/pull/61159) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* CI: fix nightly job issue [#61160](https://github.com/ClickHouse/ClickHouse/pull/61160) ([Max K.](https://github.com/maxknv)). +* Split `RangeHashedDictionary` [#61162](https://github.com/ClickHouse/ClickHouse/pull/61162) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Remove a few templates from Aggregator.cpp [#61171](https://github.com/ClickHouse/ClickHouse/pull/61171) ([Raúl Marín](https://github.com/Algunenano)). +* Avoid some logical errors in experimantal Object type [#61173](https://github.com/ClickHouse/ClickHouse/pull/61173) ([Kruglov Pavel](https://github.com/Avogar)). +* Update ReadSettings.h [#61174](https://github.com/ClickHouse/ClickHouse/pull/61174) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CI: ARM integration tests: disable tests with HDFS [#61182](https://github.com/ClickHouse/ClickHouse/pull/61182) ([Max K.](https://github.com/maxknv)). +* Disable sanitizers with 02784_parallel_replicas_automatic_decision_join [#61184](https://github.com/ClickHouse/ClickHouse/pull/61184) ([Raúl Marín](https://github.com/Algunenano)). +* Fix `02887_mutations_subcolumns` test flakiness [#61198](https://github.com/ClickHouse/ClickHouse/pull/61198) ([Nikita Taranov](https://github.com/nickitat)). +* Make variant tests a bit faster [#61199](https://github.com/ClickHouse/ClickHouse/pull/61199) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix strange log message [#61206](https://github.com/ClickHouse/ClickHouse/pull/61206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01603_insert_select_too_many_parts flakiness [#61218](https://github.com/ClickHouse/ClickHouse/pull/61218) ([Azat Khuzhin](https://github.com/azat)). +* Make every style-checker runner types scaling-out very quickly [#61231](https://github.com/ClickHouse/ClickHouse/pull/61231) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Improve `test_failed_mutations` [#61235](https://github.com/ClickHouse/ClickHouse/pull/61235) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Fix `test_merge_tree_load_parts/test.py::test_merge_tree_load_parts_corrupted` [#61236](https://github.com/ClickHouse/ClickHouse/pull/61236) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* fix `forget_partition` test [#61237](https://github.com/ClickHouse/ClickHouse/pull/61237) ([Sergei Trifonov](https://github.com/serxa)). +* Print more info in `02572_system_logs_materialized_views_ignore_errors` to debug [#61246](https://github.com/ClickHouse/ClickHouse/pull/61246) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Fix runtime error in AST Fuzzer [#61248](https://github.com/ClickHouse/ClickHouse/pull/61248) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Add retries to `02908_many_requests_to_system_replicas` [#61253](https://github.com/ClickHouse/ClickHouse/pull/61253) ([Nikita Taranov](https://github.com/nickitat)). +* Followup fix ASTRenameQuery::clone [#61254](https://github.com/ClickHouse/ClickHouse/pull/61254) ([vdimir](https://github.com/vdimir)). +* Disable test 02998_primary_key_skip_columns.sql in sanitizer builds as it can be slow [#61256](https://github.com/ClickHouse/ClickHouse/pull/61256) ([Kruglov Pavel](https://github.com/Avogar)). +* Update curl to curl with data race fix [#61264](https://github.com/ClickHouse/ClickHouse/pull/61264) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Fix `01417_freeze_partition_verbose` [#61266](https://github.com/ClickHouse/ClickHouse/pull/61266) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Free memory earlier in inserts [#61267](https://github.com/ClickHouse/ClickHouse/pull/61267) ([Anton Popov](https://github.com/CurtizJ)). +* Fixing test_build_sets_from_multiple_threads/test.py::test_set [#61286](https://github.com/ClickHouse/ClickHouse/pull/61286) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Analyzer: Fix virtual columns in StorageMerge [#61298](https://github.com/ClickHouse/ClickHouse/pull/61298) ([Dmitry Novik](https://github.com/novikd)). +* Fix 01952_optimize_distributed_group_by_sharding_key with analyzer. [#61301](https://github.com/ClickHouse/ClickHouse/pull/61301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fix data race in poco tcp server [#61309](https://github.com/ClickHouse/ClickHouse/pull/61309) ([Sema Checherinda](https://github.com/CheSema)). +* Don't use default cluster in test test_distibuted_settings [#61314](https://github.com/ClickHouse/ClickHouse/pull/61314) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix false positive assertion in cache [#61319](https://github.com/ClickHouse/ClickHouse/pull/61319) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix test test_input_format_parallel_parsing_memory_tracking [#61322](https://github.com/ClickHouse/ClickHouse/pull/61322) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix 01761_cast_to_enum_nullable with analyzer. [#61323](https://github.com/ClickHouse/ClickHouse/pull/61323) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add zookeeper retries for exists check in forcefullyRemoveBrokenOutdatedPartFromZooKeeper [#61324](https://github.com/ClickHouse/ClickHouse/pull/61324) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor changes in stress and fuzzer reports [#61333](https://github.com/ClickHouse/ClickHouse/pull/61333) ([Raúl Marín](https://github.com/Algunenano)). +* Un-flake `test_undrop_query` [#61348](https://github.com/ClickHouse/ClickHouse/pull/61348) ([Robert Schulze](https://github.com/rschu1ze)). +* Tiny improvement for replication.lib [#61361](https://github.com/ClickHouse/ClickHouse/pull/61361) ([alesapin](https://github.com/alesapin)). +* Fix bugfix check (due to "unknown object storage type: azure") [#61363](https://github.com/ClickHouse/ClickHouse/pull/61363) ([Azat Khuzhin](https://github.com/azat)). +* Fix `01599_multiline_input_and_singleline_comments` 3 minute wait [#61371](https://github.com/ClickHouse/ClickHouse/pull/61371) ([Sergei Trifonov](https://github.com/serxa)). +* Terminate EC2 on spot event if runner isn't running [#61377](https://github.com/ClickHouse/ClickHouse/pull/61377) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try fix docs check [#61378](https://github.com/ClickHouse/ClickHouse/pull/61378) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix `heap-use-after-free` for Merge table with alias [#61380](https://github.com/ClickHouse/ClickHouse/pull/61380) ([Antonio Andelic](https://github.com/antonio2368)). +* Disable `optimize_rewrite_sum_if_to_count_if` if return type is nullable (new analyzer) [#61389](https://github.com/ClickHouse/ClickHouse/pull/61389) ([Antonio Andelic](https://github.com/antonio2368)). +* Analyzer: Fix planner context for subquery in StorageMerge [#61392](https://github.com/ClickHouse/ClickHouse/pull/61392) ([Dmitry Novik](https://github.com/novikd)). +* Fix `test_failed_async_inserts` [#61394](https://github.com/ClickHouse/ClickHouse/pull/61394) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix test test_system_clusters_actual_information flakiness [#61395](https://github.com/ClickHouse/ClickHouse/pull/61395) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove default cluster from default config from test config [#61396](https://github.com/ClickHouse/ClickHouse/pull/61396) ([Raúl Marín](https://github.com/Algunenano)). +* Enable clang-tidy in headers [#61406](https://github.com/ClickHouse/ClickHouse/pull/61406) ([Robert Schulze](https://github.com/rschu1ze)). +* Add sanity check for poll_max_batch_size FileLog setting [#61408](https://github.com/ClickHouse/ClickHouse/pull/61408) ([Kruglov Pavel](https://github.com/Avogar)). +* ThreadFuzzer: randomize sleep time [#61410](https://github.com/ClickHouse/ClickHouse/pull/61410) ([Tomer Shafir](https://github.com/tomershafir)). +* Update version_date.tsv and changelogs after v23.8.11.28-lts [#61416](https://github.com/ClickHouse/ClickHouse/pull/61416) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.3.21.26-lts [#61418](https://github.com/ClickHouse/ClickHouse/pull/61418) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v24.1.7.18-stable [#61419](https://github.com/ClickHouse/ClickHouse/pull/61419) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v24.2.2.71-stable [#61420](https://github.com/ClickHouse/ClickHouse/pull/61420) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.12.5.81-stable [#61421](https://github.com/ClickHouse/ClickHouse/pull/61421) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Restore automerge for approved PRs [#61433](https://github.com/ClickHouse/ClickHouse/pull/61433) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable broken SonarCloud [#61434](https://github.com/ClickHouse/ClickHouse/pull/61434) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix `01599_multiline_input_and_singleline_comments` properly [#61440](https://github.com/ClickHouse/ClickHouse/pull/61440) ([Sergei Trifonov](https://github.com/serxa)). +* Convert test 02998_system_dns_cache_table to smoke and mirrors [#61443](https://github.com/ClickHouse/ClickHouse/pull/61443) ([vdimir](https://github.com/vdimir)). +* Check boundaries for some settings in parallel replicas [#61455](https://github.com/ClickHouse/ClickHouse/pull/61455) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use SHARD_LOAD_QUEUE_BACKLOG for dictionaries in tests [#61462](https://github.com/ClickHouse/ClickHouse/pull/61462) ([vdimir](https://github.com/vdimir)). +* Split `02125_lz4_compression_bug` [#61465](https://github.com/ClickHouse/ClickHouse/pull/61465) ([Antonio Andelic](https://github.com/antonio2368)). +* Correctly process last stacktrace in `postprocess-traces.pl` [#61470](https://github.com/ClickHouse/ClickHouse/pull/61470) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix test `test_polymorphic_parts` [#61477](https://github.com/ClickHouse/ClickHouse/pull/61477) ([Anton Popov](https://github.com/CurtizJ)). +* A definitive guide to CAST [#61491](https://github.com/ClickHouse/ClickHouse/pull/61491) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor rename in FileCache [#61494](https://github.com/ClickHouse/ClickHouse/pull/61494) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove useless code [#61498](https://github.com/ClickHouse/ClickHouse/pull/61498) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fuzzers [#61499](https://github.com/ClickHouse/ClickHouse/pull/61499) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update jdbc.md [#61506](https://github.com/ClickHouse/ClickHouse/pull/61506) ([San](https://github.com/santrancisco)). +* Fix error in clickhouse-client [#61507](https://github.com/ClickHouse/ClickHouse/pull/61507) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix clang-tidy build [#61519](https://github.com/ClickHouse/ClickHouse/pull/61519) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix infinite loop in function `hop` [#61523](https://github.com/ClickHouse/ClickHouse/pull/61523) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve tests 00159_parallel_formatting_* to to avoid timeouts [#61532](https://github.com/ClickHouse/ClickHouse/pull/61532) ([Kruglov Pavel](https://github.com/Avogar)). +* Refactoring of reading from compact parts [#61535](https://github.com/ClickHouse/ClickHouse/pull/61535) ([Anton Popov](https://github.com/CurtizJ)). +* Don't run 01459_manual_write_to_replicas in debug build as it's too slow [#61538](https://github.com/ClickHouse/ClickHouse/pull/61538) ([Kruglov Pavel](https://github.com/Avogar)). +* CI: ARM integration test - skip hdfs, kerberos, kafka [#61542](https://github.com/ClickHouse/ClickHouse/pull/61542) ([Max K.](https://github.com/maxknv)). +* More logging for loading of tables [#61546](https://github.com/ClickHouse/ClickHouse/pull/61546) ([Sergei Trifonov](https://github.com/serxa)). +* Fixing 01584_distributed_buffer_cannot_find_column with analyzer. [#61550](https://github.com/ClickHouse/ClickHouse/pull/61550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Wait for done mutation with more logs and asserts [#61554](https://github.com/ClickHouse/ClickHouse/pull/61554) ([alesapin](https://github.com/alesapin)). +* Fix read_rows count with external group by [#61555](https://github.com/ClickHouse/ClickHouse/pull/61555) ([Alexander Tokmakov](https://github.com/tavplubix)). +* queries-file should be used to specify file [#61557](https://github.com/ClickHouse/ClickHouse/pull/61557) ([danila-ermakov](https://github.com/danila-ermakov)). +* Fix `02481_async_insert_dedup_token` [#61568](https://github.com/ClickHouse/ClickHouse/pull/61568) ([Antonio Andelic](https://github.com/antonio2368)). +* Add a comment after [#61458](https://github.com/ClickHouse/ClickHouse/issues/61458) [#61580](https://github.com/ClickHouse/ClickHouse/pull/61580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix clickhouse-test client option and CLICKHOUSE_URL_PARAMS interference [#61596](https://github.com/ClickHouse/ClickHouse/pull/61596) ([vdimir](https://github.com/vdimir)). +* CI: remove compose files from integration test docker [#61597](https://github.com/ClickHouse/ClickHouse/pull/61597) ([Max K.](https://github.com/maxknv)). +* Fix 01244_optimize_distributed_group_by_sharding_key by ordering output [#61602](https://github.com/ClickHouse/ClickHouse/pull/61602) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove some tests from analyzer_tech_debt [#61603](https://github.com/ClickHouse/ClickHouse/pull/61603) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reduce header dependencies [#61604](https://github.com/ClickHouse/ClickHouse/pull/61604) ([Raúl Marín](https://github.com/Algunenano)). +* Remove some magic_enum from headers [#61606](https://github.com/ClickHouse/ClickHouse/pull/61606) ([Raúl Marín](https://github.com/Algunenano)). +* Fix configs for upgrade and bugfix [#61607](https://github.com/ClickHouse/ClickHouse/pull/61607) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add tests for multiple fuzzer issues [#61614](https://github.com/ClickHouse/ClickHouse/pull/61614) ([Raúl Marín](https://github.com/Algunenano)). +* Try to fix `02908_many_requests_to_system_replicas` again [#61616](https://github.com/ClickHouse/ClickHouse/pull/61616) ([Nikita Taranov](https://github.com/nickitat)). +* Verbose error message about analyzer_compatibility_join_using_top_level_identifier [#61631](https://github.com/ClickHouse/ClickHouse/pull/61631) ([vdimir](https://github.com/vdimir)). +* Fix 00223_shard_distributed_aggregation_memory_efficient with analyzer [#61649](https://github.com/ClickHouse/ClickHouse/pull/61649) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better fuzzer logs [#61650](https://github.com/ClickHouse/ClickHouse/pull/61650) ([Raúl Marín](https://github.com/Algunenano)). +* Fix flaky `02122_parallel_formatting_Template` [#61651](https://github.com/ClickHouse/ClickHouse/pull/61651) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix Aggregator when data is empty [#61654](https://github.com/ClickHouse/ClickHouse/pull/61654) ([Antonio Andelic](https://github.com/antonio2368)). +* Restore poco SUN files [#61655](https://github.com/ClickHouse/ClickHouse/pull/61655) ([Andy Fiddaman](https://github.com/citrus-it)). +* Another fix for `SumIfToCountIfPass` [#61656](https://github.com/ClickHouse/ClickHouse/pull/61656) ([Antonio Andelic](https://github.com/antonio2368)). +* Keeper: fix data race during snapshot destructor call [#61657](https://github.com/ClickHouse/ClickHouse/pull/61657) ([Antonio Andelic](https://github.com/antonio2368)). +* CI: integration tests: use runner as py module [#61658](https://github.com/ClickHouse/ClickHouse/pull/61658) ([Max K.](https://github.com/maxknv)). +* Fix logging of autoscaling lambda, add test for effective_capacity [#61662](https://github.com/ClickHouse/ClickHouse/pull/61662) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Small change in `DatabaseOnDisk::iterateMetadataFiles()` [#61664](https://github.com/ClickHouse/ClickHouse/pull/61664) ([Nikita Taranov](https://github.com/nickitat)). +* Build improvements by removing magic enum from header and apply some explicit template instantiation [#61665](https://github.com/ClickHouse/ClickHouse/pull/61665) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Update the dictionary for OSSFuzz [#61672](https://github.com/ClickHouse/ClickHouse/pull/61672) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Inhibit randomization in some tests and exclude some long tests from debug runs [#61676](https://github.com/ClickHouse/ClickHouse/pull/61676) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#61669](https://github.com/ClickHouse/ClickHouse/issues/61669) [#61678](https://github.com/ClickHouse/ClickHouse/pull/61678) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix use-of-uninitialized-value in HedgedConnections [#61679](https://github.com/ClickHouse/ClickHouse/pull/61679) ([Nikolay Degterinsky](https://github.com/evillique)). +* Remove clickhouse-diagnostics from the package [#61681](https://github.com/ClickHouse/ClickHouse/pull/61681) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix use-of-uninitialized-value in parseDateTimeBestEffort [#61694](https://github.com/ClickHouse/ClickHouse/pull/61694) ([Nikolay Degterinsky](https://github.com/evillique)). +* poco foundation: add illumos support [#61701](https://github.com/ClickHouse/ClickHouse/pull/61701) ([Andy Fiddaman](https://github.com/citrus-it)). +* contrib/c-ares: add illumos as a platform [#61702](https://github.com/ClickHouse/ClickHouse/pull/61702) ([Andy Fiddaman](https://github.com/citrus-it)). +* contrib/curl: Add illumos support [#61704](https://github.com/ClickHouse/ClickHouse/pull/61704) ([Andy Fiddaman](https://github.com/citrus-it)). +* Fuzzer: Try a different way to wait for the server [#61706](https://github.com/ClickHouse/ClickHouse/pull/61706) ([Raúl Marín](https://github.com/Algunenano)). +* Disable some tests for SMT [#61708](https://github.com/ClickHouse/ClickHouse/pull/61708) ([Raúl Marín](https://github.com/Algunenano)). +* Fix signal handler for sanitizer signals [#61709](https://github.com/ClickHouse/ClickHouse/pull/61709) ([Antonio Andelic](https://github.com/antonio2368)). +* Avoid `IsADirectoryError: Is a directory contrib/azure` [#61710](https://github.com/ClickHouse/ClickHouse/pull/61710) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Analyzer: fix group_by_use_nulls [#61717](https://github.com/ClickHouse/ClickHouse/pull/61717) ([Dmitry Novik](https://github.com/novikd)). +* Analyzer: Clear list of broken integration tests [#61718](https://github.com/ClickHouse/ClickHouse/pull/61718) ([Dmitry Novik](https://github.com/novikd)). +* CI: modify CI from PR body [#61725](https://github.com/ClickHouse/ClickHouse/pull/61725) ([Max K.](https://github.com/maxknv)). +* Add test for [#57820](https://github.com/ClickHouse/ClickHouse/issues/57820) [#61726](https://github.com/ClickHouse/ClickHouse/pull/61726) ([Dmitry Novik](https://github.com/novikd)). +* Revert "Revert "Un-flake test_undrop_query"" [#61727](https://github.com/ClickHouse/ClickHouse/pull/61727) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* FunctionsConversion: Start simplifying templates [#61733](https://github.com/ClickHouse/ClickHouse/pull/61733) ([Raúl Marín](https://github.com/Algunenano)). +* CI: modify it [#61735](https://github.com/ClickHouse/ClickHouse/pull/61735) ([Max K.](https://github.com/maxknv)). +* Fix segfault in SquashingTransform [#61736](https://github.com/ClickHouse/ClickHouse/pull/61736) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix DWARF format failing to skip DW_FORM_strx3 attributes [#61737](https://github.com/ClickHouse/ClickHouse/pull/61737) ([Michael Kolupaev](https://github.com/al13n321)). +* There is no such thing as broken tests [#61739](https://github.com/ClickHouse/ClickHouse/pull/61739) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Process removed files, decouple _check_mime [#61751](https://github.com/ClickHouse/ClickHouse/pull/61751) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Keeper fix: destroy `KeeperDispatcher` first [#61752](https://github.com/ClickHouse/ClickHouse/pull/61752) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix flaky `03014_async_with_dedup_part_log_rmt` [#61757](https://github.com/ClickHouse/ClickHouse/pull/61757) ([Antonio Andelic](https://github.com/antonio2368)). +* FunctionsConversion: Remove another batch of bad templates [#61773](https://github.com/ClickHouse/ClickHouse/pull/61773) ([Raúl Marín](https://github.com/Algunenano)). +* Revert "Fix bug when reading system.parts using UUID (issue 61220)." [#61774](https://github.com/ClickHouse/ClickHouse/pull/61774) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* CI: disable grpc tests on ARM [#61778](https://github.com/ClickHouse/ClickHouse/pull/61778) ([Max K.](https://github.com/maxknv)). +* Fix more tests with virtual columns in StorageMerge. [#61787](https://github.com/ClickHouse/ClickHouse/pull/61787) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove already not flaky tests with analyzer. [#61788](https://github.com/ClickHouse/ClickHouse/pull/61788) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Analyzer: Fix assert in JOIN with Distributed table [#61789](https://github.com/ClickHouse/ClickHouse/pull/61789) ([vdimir](https://github.com/vdimir)). +* A test can be slow in debug build [#61796](https://github.com/ClickHouse/ClickHouse/pull/61796) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Updated clang-19 to master. [#61798](https://github.com/ClickHouse/ClickHouse/pull/61798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test "00002_log_and_exception_messages_formatting" [#61821](https://github.com/ClickHouse/ClickHouse/pull/61821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A test is too slow for debug [#61822](https://github.com/ClickHouse/ClickHouse/pull/61822) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove DataStreams [#61824](https://github.com/ClickHouse/ClickHouse/pull/61824) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better message for logging errors [#61827](https://github.com/ClickHouse/ClickHouse/pull/61827) ([Azat Khuzhin](https://github.com/azat)). +* Fix sanitizers suppressions [#61828](https://github.com/ClickHouse/ClickHouse/pull/61828) ([Azat Khuzhin](https://github.com/azat)). +* Remove unused code [#61830](https://github.com/ClickHouse/ClickHouse/pull/61830) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove DataStreams (2) [#61831](https://github.com/ClickHouse/ClickHouse/pull/61831) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update xxhash to v0.8.2 [#61838](https://github.com/ClickHouse/ClickHouse/pull/61838) ([Shubham Ranjan](https://github.com/shubhamranjan)). +* Fix: DISTINCT in subquery with analyzer [#61847](https://github.com/ClickHouse/ClickHouse/pull/61847) ([Igor Nikonov](https://github.com/devcrafter)). +* Analyzer: fix limit/offset on shards [#61849](https://github.com/ClickHouse/ClickHouse/pull/61849) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Remove PoolBase::AllocateNewBypassingPool [#61866](https://github.com/ClickHouse/ClickHouse/pull/61866) ([Azat Khuzhin](https://github.com/azat)). +* Try to fix 02901_parallel_replicas_rollup with analyzer. [#61875](https://github.com/ClickHouse/ClickHouse/pull/61875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add test for [#57808](https://github.com/ClickHouse/ClickHouse/issues/57808) [#61879](https://github.com/ClickHouse/ClickHouse/pull/61879) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* CI: merge queue support [#61881](https://github.com/ClickHouse/ClickHouse/pull/61881) ([Max K.](https://github.com/maxknv)). +* Update create.sql [#61885](https://github.com/ClickHouse/ClickHouse/pull/61885) ([Kseniia Sumarokova](https://github.com/kssenii)). +* no smaller unit in date_trunc [#61888](https://github.com/ClickHouse/ClickHouse/pull/61888) ([jsc0218](https://github.com/jsc0218)). +* Move KQL trash where it is supposed to be [#61903](https://github.com/ClickHouse/ClickHouse/pull/61903) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Changelog for 24.3 [#61909](https://github.com/ClickHouse/ClickHouse/pull/61909) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v23.3.22.3-lts [#61914](https://github.com/ClickHouse/ClickHouse/pull/61914) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.8.12.13-lts [#61915](https://github.com/ClickHouse/ClickHouse/pull/61915) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* No "please" [#61916](https://github.com/ClickHouse/ClickHouse/pull/61916) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v23.12.6.19-stable [#61917](https://github.com/ClickHouse/ClickHouse/pull/61917) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v24.1.8.22-stable [#61918](https://github.com/ClickHouse/ClickHouse/pull/61918) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Fix flaky test_broken_projestions/test.py::test_broken_ignored_replic… [#61932](https://github.com/ClickHouse/ClickHouse/pull/61932) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Check is Rust avaiable for build, if not, suggest a way to disable Rust support [#61938](https://github.com/ClickHouse/ClickHouse/pull/61938) ([Azat Khuzhin](https://github.com/azat)). +* CI: new ci menu in PR body [#61948](https://github.com/ClickHouse/ClickHouse/pull/61948) ([Max K.](https://github.com/maxknv)). +* Remove flaky test `01193_metadata_loading` [#61961](https://github.com/ClickHouse/ClickHouse/pull/61961) ([Nikita Taranov](https://github.com/nickitat)). + +#### Packaging Improvement + +* Adding the `--now` option to enable and start service automatically when installing the database server completely. [#60656](https://github.com/ClickHouse/ClickHouse/pull/60656) ([Chun-Sheng, Li](https://github.com/peter279k)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index e4482998fd6..ca1a23a99db 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v24.3.1.2672-lts 2024-03-27 v24.2.2.71-stable 2024-03-15 v24.2.1.2248-stable 2024-02-29 v24.1.8.22-stable 2024-03-26 From 6a585600f0d8ce31201fa909d514ac1a913b0325 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 27 Mar 2024 19:21:13 +0100 Subject: [PATCH 0709/1165] Update test --- tests/integration/test_broken_projections/test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py index 5a6232819b6..adb7b58f171 100644 --- a/tests/integration/test_broken_projections/test.py +++ b/tests/integration/test_broken_projections/test.py @@ -728,10 +728,13 @@ def test_mutation_with_broken_projection(cluster): # but all parts apart from the first have only hardlinks to files in previous part. assert ["all_0_0_0_5", "all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == get_parts( node, table_name - ) + ) or ["all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == get_parts(node, table_name) # Still broken because it was hardlinked. - assert "all_2_2_0_5" in get_broken_projections_info(node, table_name) + broken = get_broken_projections_info(node, table_name) + assert ( + "all_2_2_0_5" in broken or "" == broken + ) # second could be because of a merge. check(node, table_name, 0) From 2d36f6240e81bf0b7b2064360497e27fc76a0f49 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 27 Mar 2024 19:25:04 +0100 Subject: [PATCH 0710/1165] Update src/Databases/DatabaseOnDisk.cpp --- src/Databases/DatabaseOnDisk.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 5ea2a1a5421..49b10693331 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -75,7 +75,7 @@ std::pair createTableFromAST( auto table_function = factory.get(table_function_ast, context); ColumnsDescription columns; if (ast_create_query.columns_list && ast_create_query.columns_list->columns) - columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, LoadingStrictnessLevel::ATTACH); + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, mode); StoragePtr storage = table_function->execute(table_function_ast, context, ast_create_query.getTable(), std::move(columns)); storage->renameInMemory(ast_create_query); return {ast_create_query.getTable(), storage}; From 920ba121febb0cb5f823d4b4d3897e4079bf1fcf Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 27 Mar 2024 19:17:41 +0000 Subject: [PATCH 0711/1165] Fix scalars for CREATE AS SELECT for analyzer. --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 13 +++++++------ .../03032_scalars_create_as_select.reference | 1 + .../0_stateless/03032_scalars_create_as_select.sql | 2 ++ 3 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/03032_scalars_create_as_select.reference create mode 100644 tests/queries/0_stateless/03032_scalars_create_as_select.sql diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index afc51d27a0b..b2fc74cafd4 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2059,13 +2059,9 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden subquery_context->setSetting("use_structure_from_insertion_table_in_table_functions", false); auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_subquery*/); + options.only_analyze = only_analyze; auto interpreter = std::make_unique(node->toAST(), subquery_context, subquery_context->getViewSource(), options); - auto io = interpreter->execute(); - PullingAsyncPipelineExecutor executor(io.pipeline); - io.pipeline.setProgressCallback(context->getProgressCallback()); - io.pipeline.setProcessListElement(context->getProcessListElement()); - if (only_analyze) { /// If query is only analyzed, then constants are not correct. @@ -2082,6 +2078,11 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden } else { + auto io = interpreter->execute(); + PullingAsyncPipelineExecutor executor(io.pipeline); + io.pipeline.setProgressCallback(context->getProgressCallback()); + io.pipeline.setProcessListElement(context->getProcessListElement()); + Block block; while (block.rows() == 0 && executor.pull(block)) @@ -2193,7 +2194,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden auto & nearest_query_scope_query_node = nearest_query_scope->scope_node->as(); auto & mutable_context = nearest_query_scope_query_node.getMutableContext(); - auto scalar_query_hash_string = DB::toString(node_with_hash.hash); + auto scalar_query_hash_string = DB::toString(node_with_hash.hash) + (only_analyze ? "_analyze" : ""); if (mutable_context->hasQueryContext()) mutable_context->getQueryContext()->addScalar(scalar_query_hash_string, scalar_block); diff --git a/tests/queries/0_stateless/03032_scalars_create_as_select.reference b/tests/queries/0_stateless/03032_scalars_create_as_select.reference new file mode 100644 index 00000000000..2e73a0708a2 --- /dev/null +++ b/tests/queries/0_stateless/03032_scalars_create_as_select.reference @@ -0,0 +1 @@ +['a','b'] diff --git a/tests/queries/0_stateless/03032_scalars_create_as_select.sql b/tests/queries/0_stateless/03032_scalars_create_as_select.sql new file mode 100644 index 00000000000..5b20b8cf6f9 --- /dev/null +++ b/tests/queries/0_stateless/03032_scalars_create_as_select.sql @@ -0,0 +1,2 @@ +create or replace table query_run_metric_arrays engine Memory as with (with (select groupUniqArrayArray(['a', 'b']) from numbers(1)) as all_names select all_names) as all_metrics select all_metrics; +select * from query_run_metric_arrays; From 043c625898883259011b755317975ee880cac692 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Mar 2024 19:19:08 +0000 Subject: [PATCH 0712/1165] Don't treat Bool and number variants as suspicious --- src/Interpreters/parseColumnsListForTableFunction.cpp | 4 ++++ .../03032_variant_bool_number_not_suspicious.reference | 1 + .../0_stateless/03032_variant_bool_number_not_suspicious.sql | 4 ++++ 3 files changed, 9 insertions(+) create mode 100644 tests/queries/0_stateless/03032_variant_bool_number_not_suspicious.reference create mode 100644 tests/queries/0_stateless/03032_variant_bool_number_not_suspicious.sql diff --git a/src/Interpreters/parseColumnsListForTableFunction.cpp b/src/Interpreters/parseColumnsListForTableFunction.cpp index 30a41c090d5..498c6dc4c84 100644 --- a/src/Interpreters/parseColumnsListForTableFunction.cpp +++ b/src/Interpreters/parseColumnsListForTableFunction.cpp @@ -86,6 +86,10 @@ void validateDataType(const DataTypePtr & type_to_check, const DataTypeValidatio { for (size_t j = i + 1; j < variants.size(); ++j) { + /// Don't consider bool as similar to something (like number). + if (isBool(variants[i]) || isBool(variants[j])) + continue; + if (auto supertype = tryGetLeastSupertype(DataTypes{variants[i], variants[j]})) { throw Exception( diff --git a/tests/queries/0_stateless/03032_variant_bool_number_not_suspicious.reference b/tests/queries/0_stateless/03032_variant_bool_number_not_suspicious.reference new file mode 100644 index 00000000000..27ba77ddaf6 --- /dev/null +++ b/tests/queries/0_stateless/03032_variant_bool_number_not_suspicious.reference @@ -0,0 +1 @@ +true diff --git a/tests/queries/0_stateless/03032_variant_bool_number_not_suspicious.sql b/tests/queries/0_stateless/03032_variant_bool_number_not_suspicious.sql new file mode 100644 index 00000000000..b7609d34f58 --- /dev/null +++ b/tests/queries/0_stateless/03032_variant_bool_number_not_suspicious.sql @@ -0,0 +1,4 @@ +set allow_experimental_variant_type=1; +set allow_suspicious_variant_types=0; +select 'true'::Bool::Variant(UInt32, Bool); + From 3ce122ef2061ed7b40766b1bf0211309b89fa621 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 27 Mar 2024 19:23:16 +0000 Subject: [PATCH 0713/1165] Fixing tests. --- src/Analyzer/HashUtils.h | 4 ++-- src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Analyzer/HashUtils.h b/src/Analyzer/HashUtils.h index 8673361d499..e0b96ea9e58 100644 --- a/src/Analyzer/HashUtils.h +++ b/src/Analyzer/HashUtils.h @@ -52,8 +52,8 @@ using QueryTreeNodeConstRawPtrWithHashMap = std::unordered_map -struct std::hash> +template +struct std::hash> { size_t operator()(const DB::QueryTreeNodeWithHash & node_with_hash) const { diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index a1832b2db3f..8b687922288 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -776,7 +776,7 @@ struct IdentifierResolveScope /// Table expression node to data std::unordered_map table_expression_node_to_data; - QueryTreeNodePtrWithHashSet nullable_group_by_keys; + QueryTreeNodePtrWithHashWithoutAliasSet nullable_group_by_keys; /// Here we count the number of nullable GROUP BY keys we met resolving expression. /// E.g. for a query `SELECT tuple(tuple(number)) FROM numbers(10) GROUP BY (number, tuple(number)) with cube` /// both `number` and `tuple(number)` would be in nullable_group_by_keys. From ce2e5a7d13c2e750f8b7d7fecb4a644b2568fc42 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 27 Mar 2024 19:25:07 +0000 Subject: [PATCH 0714/1165] Fixing tests. --- src/Analyzer/HashUtils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/HashUtils.h b/src/Analyzer/HashUtils.h index e0b96ea9e58..eb6aac88fe9 100644 --- a/src/Analyzer/HashUtils.h +++ b/src/Analyzer/HashUtils.h @@ -55,7 +55,7 @@ using QueryTreeNodeConstRawPtrWithHashMap = std::unordered_map struct std::hash> { - size_t operator()(const DB::QueryTreeNodeWithHash & node_with_hash) const + size_t operator()(const DB::QueryTreeNodeWithHash & node_with_hash) const { return node_with_hash.hash.low64; } From 343b51ccfa13c25648a6004954b9442f8de3f86b Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 27 Mar 2024 21:27:07 +0100 Subject: [PATCH 0715/1165] fix style --- tests/integration/test_quorum_inserts/test.py | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py index 0cc53558521..b842a54741e 100644 --- a/tests/integration/test_quorum_inserts/test.py +++ b/tests/integration/test_quorum_inserts/test.py @@ -364,8 +364,11 @@ def test_insert_quorum_with_ttl(started_cluster): zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_ttl ON CLUSTER cluster") + def test_insert_quorum_with_keeper_loss_connection(): - zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_keeper_fail ON CLUSTER cluster") + zero.query( + "DROP TABLE IF EXISTS test_insert_quorum_with_keeper_fail ON CLUSTER cluster" + ) create_query = ( "CREATE TABLE test_insert_quorum_with_keeper_loss" "(a Int8, d Date) " @@ -373,19 +376,15 @@ def test_insert_quorum_with_keeper_loss_connection(): "ORDER BY a " ) - print("Create Replicated table with two replicas") zero.query(create_query) first.query(create_query) - print("Stop fetches for test_insert_quorum_with_keeper_loss at first replica.") first.query("SYSTEM STOP FETCHES test_insert_quorum_with_keeper_loss") - print("Inject insert fail and retry pause for server zero") zero.query("SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op") zero.query("SYSTEM ENABLE FAILPOINT replicated_merge_tree_insert_retry_pause") with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - print("Inject zero") insert_future = executor.submit( lambda: zero.query( "INSERT INTO test_insert_quorum_with_keeper_loss(a,d) VALUES(1, '2011-01-01')", @@ -393,11 +392,9 @@ def test_insert_quorum_with_keeper_loss_connection(): ) ) - print("Inject zero keeper") pm = PartitionManager() pm.drop_instance_zk_connections(zero) - print("Wait zero is not active") retries = 0 zk = cluster.get_kazoo_client("zoo1") while True: @@ -414,32 +411,33 @@ def test_insert_quorum_with_keeper_loss_connection(): if retries == 120: raise Exception("Can not wait cluster replica inactive") - print("Inject first wait for quorum fail") first.query("SYSTEM ENABLE FAILPOINT finish_set_quorum_failed_parts") quorum_fail_future = executor.submit( - lambda: first.query("SYSTEM WAIT FAILPOINT finish_set_quorum_failed_parts", timeout=300) + lambda: first.query( + "SYSTEM WAIT FAILPOINT finish_set_quorum_failed_parts", timeout=300 + ) ) - print("Start fetches at first node") first.query("SYSTEM START FETCHES test_insert_quorum_with_keeper_loss") concurrent.futures.wait([quorum_fail_future]) - assert(quorum_fail_future.exception() is None) + assert quorum_fail_future.exception() is None - print("Inject zero wait for clean quorum fail parts") zero.query("SYSTEM ENABLE FAILPOINT finish_clean_quorum_failed_parts") clean_quorum_fail_parts_future = executor.submit( - lambda: first.query("SYSTEM WAIT FAILPOINT finish_clean_quorum_failed_parts", timeout=300) + lambda: first.query( + "SYSTEM WAIT FAILPOINT finish_clean_quorum_failed_parts", timeout=300 + ) ) - print("Restore zero keeper") pm.restore_instance_zk_connections(zero) concurrent.futures.wait([clean_quorum_fail_parts_future]) - assert(clean_quorum_fail_parts_future.exception() is None) + assert clean_quorum_fail_parts_future.exception() is None - print("Disable retry pause") zero.query("SYSTEM DISABLE FAILPOINT replicated_merge_tree_insert_retry_pause") concurrent.futures.wait([insert_future]) - assert(insert_future.exception() is not None) - assert(not zero.contains_in_log("LOGICAL_ERROR")) - assert(zero.contains_in_log("fails to commit and will not retry or clean garbage")) + assert insert_future.exception() is not None + assert not zero.contains_in_log("LOGICAL_ERROR") + assert zero.contains_in_log( + "fails to commit and will not retry or clean garbage" + ) From df066193990042d35fcfabb58cd33412693fe8ca Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Mar 2024 20:40:26 +0000 Subject: [PATCH 0716/1165] Better conversion from String to Variant --- .../Serializations/SerializationBool.cpp | 2 +- src/DataTypes/getLeastSupertype.cpp | 3 + src/Functions/CastOverloadResolver.cpp | 6 +- src/Functions/FunctionsConversion.cpp | 85 +++++++++++++------ .../0_stateless/02941_variant_type_1.sh | 6 +- .../0_stateless/02941_variant_type_2.sh | 2 +- .../0_stateless/02941_variant_type_3.sh | 2 +- .../0_stateless/02941_variant_type_4.sh | 2 +- .../0_stateless/02942_variant_cast.reference | 2 +- .../03032_string_to_variant_cast.reference | 13 +++ .../03032_string_to_variant_cast.sql | 17 ++++ 11 files changed, 107 insertions(+), 33 deletions(-) create mode 100644 tests/queries/0_stateless/03032_string_to_variant_cast.reference create mode 100644 tests/queries/0_stateless/03032_string_to_variant_cast.sql diff --git a/src/DataTypes/Serializations/SerializationBool.cpp b/src/DataTypes/Serializations/SerializationBool.cpp index f745fac4d30..d6a74e5cb8f 100644 --- a/src/DataTypes/Serializations/SerializationBool.cpp +++ b/src/DataTypes/Serializations/SerializationBool.cpp @@ -194,12 +194,12 @@ ReturnType deserializeImpl( buf.dropCheckpoint(); if (buf.hasUnreadData()) { + restore_column_if_needed(); if constexpr (throw_exception) throw Exception( ErrorCodes::CANNOT_PARSE_BOOL, "Cannot continue parsing after parsed bool value because it will result in the loss of some data. It may happen if " "bool_true_representation or bool_false_representation contains some delimiters of input format"); - restore_column_if_needed(); return ReturnType(false); } return ReturnType(true); diff --git a/src/DataTypes/getLeastSupertype.cpp b/src/DataTypes/getLeastSupertype.cpp index dec77119eed..e69b0411aac 100644 --- a/src/DataTypes/getLeastSupertype.cpp +++ b/src/DataTypes/getLeastSupertype.cpp @@ -463,6 +463,9 @@ DataTypePtr getLeastSupertype(const DataTypes & types) /// nested_type will be nullptr, we should return nullptr in this case. if (!nested_type) return nullptr; + /// Common type for Nullable(Nothing) and Variant(...) is Variant(...) + if (isVariant(nested_type)) + return nested_type; return std::make_shared(nested_type); } } diff --git a/src/Functions/CastOverloadResolver.cpp b/src/Functions/CastOverloadResolver.cpp index 5ca4b0bc579..0f54ff52ba2 100644 --- a/src/Functions/CastOverloadResolver.cpp +++ b/src/Functions/CastOverloadResolver.cpp @@ -100,7 +100,11 @@ protected: validateDataType(type, data_type_validation_settings); if (cast_type == CastType::accurateOrNull) - return makeNullable(type); + { + /// Variant handles NULLs by itself during conversions. + if (!isVariant(type)) + return makeNullable(type); + } if (internal) return type; diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 7049ca44110..6a1a2d5e888 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -1815,6 +1815,7 @@ struct ConvertImpl /// Generic conversion of any type from String. Used for complex types: Array and Tuple or types with custom serialization. +template struct ConvertImplGenericFromString { static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * column_nullable, size_t input_rows_count) @@ -1854,29 +1855,34 @@ struct ConvertImplGenericFromString { serialization_from.deserializeWholeText(column_to, read_buffer, format_settings); } - catch (const Exception & e) + catch (const Exception &) { - auto * nullable_column = typeid_cast(&column_to); - if (e.code() == ErrorCodes::CANNOT_PARSE_BOOL && nullable_column) - { - auto & col_nullmap = nullable_column->getNullMapData(); - if (col_nullmap.size() != nullable_column->size()) - col_nullmap.resize_fill(nullable_column->size()); - if (nullable_column->size() == (i + 1)) - nullable_column->popBack(1); - nullable_column->insertDefault(); - continue; - } - throw; + if constexpr (throw_on_error) + throw; + /// Check if exception happened after we inserted the value + /// (deserializeWholeText should not do it, but let's check anyway). + if (column_to.size() > i) + column_to.popBack(column_to.size() - i); + column_to.insertDefault(); } + /// Usually deserializeWholeText checks for eof after parsing, but let's check one more time just in case. if (!read_buffer.eof()) { - if (result_type) - throwExceptionForIncompletelyParsedValue(read_buffer, *result_type); + if constexpr (throw_on_error) + { + if (result_type) + throwExceptionForIncompletelyParsedValue(read_buffer, *result_type); + else + throw Exception( + ErrorCodes::CANNOT_PARSE_TEXT, "Cannot parse string to column {}. Expected eof", column_to.getName()); + } else - throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, - "Cannot parse string to column {}. Expected eof", column_to.getName()); + { + if (column_to.size() > i) + column_to.popBack(column_to.size() - i); + column_to.insertDefault(); + } } } } @@ -3279,7 +3285,9 @@ private: { if (checkAndGetDataType(from_type.get())) { - return &ConvertImplGenericFromString::execute; + if (cast_type == CastType::accurateOrNull) + return &ConvertImplGenericFromString::execute; + return &ConvertImplGenericFromString::execute; } return createWrapper(from_type, to_type, requested_result_is_nullable); @@ -3442,7 +3450,7 @@ private: /// Conversion from String through parsing. if (checkAndGetDataType(from_type_untyped.get())) { - return &ConvertImplGenericFromString::execute; + return &ConvertImplGenericFromString::execute; } else if (const auto * agg_type = checkAndGetDataType(from_type_untyped.get())) { @@ -3485,7 +3493,7 @@ private: /// Conversion from String through parsing. if (checkAndGetDataType(from_type_untyped.get())) { - return &ConvertImplGenericFromString::execute; + return &ConvertImplGenericFromString::execute; } DataTypePtr from_type_holder; @@ -3576,7 +3584,7 @@ private: /// Conversion from String through parsing. if (checkAndGetDataType(from_type_untyped.get())) { - return &ConvertImplGenericFromString::execute; + return &ConvertImplGenericFromString::execute; } const auto * from_type = checkAndGetDataType(from_type_untyped.get()); @@ -3921,7 +3929,7 @@ private: { return [] (ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * nullable_source, size_t input_rows_count) { - auto res = ConvertImplGenericFromString::execute(arguments, result_type, nullable_source, input_rows_count)->assumeMutable(); + auto res = ConvertImplGenericFromString::execute(arguments, result_type, nullable_source, input_rows_count)->assumeMutable(); res->finalize(); return res; }; @@ -4076,6 +4084,29 @@ private: return ColumnVariant::create(discriminators, variants); } + WrapperType createStringToVariantWrapper() const + { + return [&](ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable *, size_t input_rows_count) -> ColumnPtr + { + auto column = arguments[0].column->convertToFullColumnIfLowCardinality(); + auto args = arguments; + args[0].column = column; + + const ColumnNullable * column_nullable = nullptr; + if (isColumnNullable(*args[0].column)) + { + column_nullable = assert_cast(args[0].column.get()); + args[0].column = column_nullable->getNestedColumnPtr(); + } + + args[0].type = removeNullable(removeLowCardinality(args[0].type)); + + if (cast_type == CastType::accurateOrNull) + return ConvertImplGenericFromString::execute(args, result_type, column_nullable, input_rows_count); + return ConvertImplGenericFromString::execute(args, result_type, column_nullable, input_rows_count); + }; + } + WrapperType createColumnToVariantWrapper(const DataTypePtr & from_type, const DataTypeVariant & to_variant) const { /// We allow converting NULL to Variant(...) as Variant can store NULLs. @@ -4089,6 +4120,9 @@ private: }; } + if (isStringOrFixedString(removeNullable(removeLowCardinality(from_type)))) + return createStringToVariantWrapper(); + auto variant_discr_opt = to_variant.tryGetVariantDiscriminator(*removeNullableOrLowCardinalityNullable(from_type)); if (!variant_discr_opt) throw Exception(ErrorCodes::CANNOT_CONVERT_TYPE, "Cannot convert type {} to {}. Conversion to Variant allowed only for types from this Variant", from_type->getName(), to_variant.getName()); @@ -4691,7 +4725,7 @@ private: if (to_type->getCustomSerialization() && to_type->getCustomName()) { - ret = [requested_result_is_nullable]( + ret = [this, requested_result_is_nullable]( ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * column_nullable, @@ -4700,7 +4734,10 @@ private: auto wrapped_result_type = result_type; if (requested_result_is_nullable) wrapped_result_type = makeNullable(result_type); - return ConvertImplGenericFromString::execute( + if (this->cast_type == CastType::accurateOrNull) + return ConvertImplGenericFromString::execute( + arguments, wrapped_result_type, column_nullable, input_rows_count); + return ConvertImplGenericFromString::execute( arguments, wrapped_result_type, column_nullable, input_rows_count); }; return true; diff --git a/tests/queries/0_stateless/02941_variant_type_1.sh b/tests/queries/0_stateless/02941_variant_type_1.sh index 773a8c4a5e4..4fb76532a05 100755 --- a/tests/queries/0_stateless/02941_variant_type_1.sh +++ b/tests/queries/0_stateless/02941_variant_type_1.sh @@ -14,7 +14,7 @@ function test1_insert() echo "test1 insert" $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number from numbers(3); -insert into test select number + 6, 'str_' || toString(number) from numbers(3); +insert into test select number + 6, ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(3); insert into test select number + 12, tuple(number, number + 1)::Tuple(a UInt32, b UInt32) from numbers(3); insert into test select number + 15, range(number + 1)::Array(UInt64) from numbers(3);" @@ -40,7 +40,7 @@ function test2_insert() echo "test2 insert" $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number % 2 ? NULL : number from numbers(3); -insert into test select number + 6, number % 2 ? NULL : 'str_' || toString(number) from numbers(3); +insert into test select number + 6, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(('lc_str_' || toString(number))::LowCardinality(String), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(3); insert into test select number + 12, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(tuple(number, number + 1)::Tuple(a UInt32, b UInt32), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(3); insert into test select number + 15, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(range(number + 1)::Array(UInt64), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(3);" @@ -64,7 +64,7 @@ select v.\`Array(UInt64)\`.size0 from test order by id;" function test3_insert() { echo "test3 insert" - $CH_CLIENT -q "insert into test with 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))' as type select number, multiIf(number % 6 == 0, CAST(NULL, type), number % 6 == 1, CAST('str_' || toString(number), type), number % 6 == 2, CAST(number, type), number % 6 == 3, CAST(('lc_str_' || toString(number))::LowCardinality(String), type), number % 6 == 4, CAST(tuple(number, number + 1)::Tuple(a UInt32, b UInt32), type), CAST(range(number + 1)::Array(UInt64), type)) as res from numbers(18);" + $CH_CLIENT -q "insert into test with 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))' as type select number, multiIf(number % 6 == 0, CAST(NULL, type), number % 6 == 1, CAST(('str_' || toString(number))::Variant(String), type), number % 6 == 2, CAST(number, type), number % 6 == 3, CAST(('lc_str_' || toString(number))::LowCardinality(String), type), number % 6 == 4, CAST(tuple(number, number + 1)::Tuple(a UInt32, b UInt32), type), CAST(range(number + 1)::Array(UInt64), type)) as res from numbers(18);" } function test3_select() diff --git a/tests/queries/0_stateless/02941_variant_type_2.sh b/tests/queries/0_stateless/02941_variant_type_2.sh index d1fa0a777c9..995b622b6bf 100755 --- a/tests/queries/0_stateless/02941_variant_type_2.sh +++ b/tests/queries/0_stateless/02941_variant_type_2.sh @@ -14,7 +14,7 @@ function test4_insert() echo "test4 insert" $CH_CLIENT -nmq "insert into test select number, NULL from numbers(100000); insert into test select number + 100000, number from numbers(100000); -insert into test select number + 200000, 'str_' || toString(number) from numbers(100000); +insert into test select number + 200000, ('str_' || toString(number))::Variant(String) from numbers(100000); insert into test select number + 300000, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(100000); insert into test select number + 400000, tuple(number, number + 1)::Tuple(a UInt32, b UInt32) from numbers(100000); insert into test select number + 500000, range(number % 20 + 1)::Array(UInt64) from numbers(100000);" diff --git a/tests/queries/0_stateless/02941_variant_type_3.sh b/tests/queries/0_stateless/02941_variant_type_3.sh index a0efead280a..9fbdf6de8a7 100755 --- a/tests/queries/0_stateless/02941_variant_type_3.sh +++ b/tests/queries/0_stateless/02941_variant_type_3.sh @@ -15,7 +15,7 @@ function test5_insert() $CH_CLIENT -nmq " insert into test select number, NULL from numbers(200000); insert into test select number + 200000, number % 2 ? NULL : number from numbers(200000); -insert into test select number + 400000, number % 2 ? NULL : 'str_' || toString(number) from numbers(200000); +insert into test select number + 400000, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(200000); insert into test select number + 600000, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(('lc_str_' || toString(number))::LowCardinality(String), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(200000); insert into test select number + 800000, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(tuple(number, number + 1)::Tuple(a UInt32, b UInt32), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(200000); insert into test select number + 1000000, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(range(number % 20 + 1)::Array(UInt64), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(200000);" diff --git a/tests/queries/0_stateless/02941_variant_type_4.sh b/tests/queries/0_stateless/02941_variant_type_4.sh index 336540d1e79..f6eaf2fcc9a 100755 --- a/tests/queries/0_stateless/02941_variant_type_4.sh +++ b/tests/queries/0_stateless/02941_variant_type_4.sh @@ -12,7 +12,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test6_insert() { echo "test6 insert" - $CH_CLIENT -q "insert into test with 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))' as type select number, multiIf(number % 6 == 0, CAST(NULL, type), number % 6 == 1, CAST('str_' || toString(number), type), number % 6 == 2, CAST(number, type), number % 6 == 3, CAST(('lc_str_' || toString(number))::LowCardinality(String), type), number % 6 == 4, CAST(tuple(number, number + 1)::Tuple(a UInt32, b UInt32), type), CAST(range(number % 20 + 1)::Array(UInt64), type)) as res from numbers(1200000);" + $CH_CLIENT -q "insert into test with 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))' as type select number, multiIf(number % 6 == 0, CAST(NULL, type), number % 6 == 1, CAST(('str_' || toString(number))::Variant(String), type), number % 6 == 2, CAST(number, type), number % 6 == 3, CAST(('lc_str_' || toString(number))::LowCardinality(String), type), number % 6 == 4, CAST(tuple(number, number + 1)::Tuple(a UInt32, b UInt32), type), CAST(range(number % 20 + 1)::Array(UInt64), type)) as res from numbers(1200000);" } function test6_select() diff --git a/tests/queries/0_stateless/02942_variant_cast.reference b/tests/queries/0_stateless/02942_variant_cast.reference index f3fd7a9ba33..d300ca655a6 100644 --- a/tests/queries/0_stateless/02942_variant_cast.reference +++ b/tests/queries/0_stateless/02942_variant_cast.reference @@ -6,7 +6,7 @@ \N Hello Hello -NULL +\N Hello Hello \N diff --git a/tests/queries/0_stateless/03032_string_to_variant_cast.reference b/tests/queries/0_stateless/03032_string_to_variant_cast.reference new file mode 100644 index 00000000000..6531e2206a5 --- /dev/null +++ b/tests/queries/0_stateless/03032_string_to_variant_cast.reference @@ -0,0 +1,13 @@ +42 UInt64 +abc String +\N None +[1,2,3] Array(UInt64) +[1, 2, 3 String +\N None +42 UInt64 +42 UInt64 +42 UInt64 +\N None +42 UInt64 +\N None +\N None diff --git a/tests/queries/0_stateless/03032_string_to_variant_cast.sql b/tests/queries/0_stateless/03032_string_to_variant_cast.sql new file mode 100644 index 00000000000..67a501b9628 --- /dev/null +++ b/tests/queries/0_stateless/03032_string_to_variant_cast.sql @@ -0,0 +1,17 @@ +set allow_experimental_variant_type=1; +select CAST('42', 'Variant(String, UInt64)') as v, variantType(v); +select CAST('abc', 'Variant(String, UInt64)') as v, variantType(v); +select CAST('null', 'Variant(String, UInt64)') as v, variantType(v); +select CAST('[1, 2, 3]', 'Variant(String, Array(UInt64))') as v, variantType(v); +select CAST('[1, 2, 3', 'Variant(String, Array(UInt64))') as v, variantType(v); +select CAST('42', 'Variant(Date)') as v, variantType(v); -- {serverError INCORRECT_DATA} +select accurateCastOrNull('42', 'Variant(Date)') as v, variantType(v); + +select CAST('42'::FixedString(2), 'Variant(String, UInt64)') as v, variantType(v); +select CAST('42'::LowCardinality(String), 'Variant(String, UInt64)') as v, variantType(v); +select CAST('42'::Nullable(String), 'Variant(String, UInt64)') as v, variantType(v); +select CAST(NULL::Nullable(String), 'Variant(String, UInt64)') as v, variantType(v); +select CAST('42'::LowCardinality(Nullable(String)), 'Variant(String, UInt64)') as v, variantType(v); +select CAST(NULL::LowCardinality(Nullable(String)), 'Variant(String, UInt64)') as v, variantType(v); +select CAST(NULL::LowCardinality(Nullable(FixedString(2))), 'Variant(String, UInt64)') as v, variantType(v); + From ab327e69c525eb766688f67c7aae7f04bb14de81 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Mar 2024 20:53:52 +0000 Subject: [PATCH 0717/1165] Add docs about converting from string --- docs/en/sql-reference/data-types/variant.md | 57 ++++++++++++++++++--- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/data-types/variant.md b/docs/en/sql-reference/data-types/variant.md index 7d10d4b0e97..24923e4d30f 100644 --- a/docs/en/sql-reference/data-types/variant.md +++ b/docs/en/sql-reference/data-types/variant.md @@ -192,20 +192,65 @@ SELECT toTypeName(variantType(v)) FROM test LIMIT 1; ## Conversion between Variant column and other columns -There are 3 possible conversions that can be performed with Variant column. +There are 4 possible conversions that can be performed with Variant column. + +### Converting String column to a Variant column + +Conversion from `String` to `Variant` is performed by parsing a value of `Variant` type from the string value: + +```sql +SELECT '42'::Variant(String, UInt64) as variant, variantType(variant) as variant_type +``` + +```text +┌─variant─┬─variant_type─┐ +│ 42 │ UInt64 │ +└─────────┴──────────────┘ +``` + +```sql +SELECT '[1, 2, 3]'::Variant(String, Array(UInt64)) as variant, variantType(variant) as variant_type +``` + +```text +┌─variant─┬─variant_type──┐ +│ [1,2,3] │ Array(UInt64) │ +└─────────┴───────────────┘ +``` + +```sql +SELECT CAST(map('key1', '42', 'key2', 'true', 'key3', '2020-01-01'), 'Map(String, Variant(UInt64, Bool, Date))') as map_of_variants, mapApply((k, v) -> (k, variantType(v)), map_of_variants) as map_of_variant_types``` +``` + +```text +┌─map_of_variants─────────────────────────────┬─map_of_variant_types──────────────────────────┐ +│ {'key1':42,'key2':true,'key3':'2020-01-01'} │ {'key1':'UInt64','key2':'Bool','key3':'Date'} │ +└─────────────────────────────────────────────┴───────────────────────────────────────────────┘ +``` ### Converting an ordinary column to a Variant column It is possible to convert ordinary column with type `T` to a `Variant` column containing this type: ```sql -SELECT toTypeName(variant) as type_name, 'Hello, World!'::Variant(UInt64, String, Array(UInt64)) as variant; -``` +SELECT toTypeName(variant) as type_name, [1,2,3]::Array(UInt64)::Variant(UInt64, String, Array(UInt64)) as variant, variantType(variant) as variant_name + ``` ```text -┌─type_name──────────────────────────────┬─variant───────┐ -│ Variant(Array(UInt64), String, UInt64) │ Hello, World! │ -└────────────────────────────────────────┴───────────────┘ +┌─type_name──────────────────────────────┬─variant─┬─variant_name──┐ +│ Variant(Array(UInt64), String, UInt64) │ [1,2,3] │ Array(UInt64) │ +└────────────────────────────────────────┴─────────┴───────────────┘ +``` + +Note: converting from `String` type is always performed through parsing, if you need to convert `String` column to `String` variant of a `Variant` without parsing, you can do the following: +```sql +SELECT '[1, 2, 3]'::Variant(String)::Variant(String, Array(UInt64), UInt64) as variant, variantType(variant) as variant_type +``` + +```sql +┌─variant───┬─variant_type─┐ +│ [1, 2, 3] │ String │ +└───────────┴──────────────┘ ``` ### Converting a Variant column to an ordinary column From d2e375ce33778d8fe66290f46353ac8b8a1f8187 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Mar 2024 20:55:56 +0000 Subject: [PATCH 0718/1165] Better docs --- docs/en/sql-reference/data-types/variant.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/data-types/variant.md b/docs/en/sql-reference/data-types/variant.md index 24923e4d30f..e41f3c3119f 100644 --- a/docs/en/sql-reference/data-types/variant.md +++ b/docs/en/sql-reference/data-types/variant.md @@ -190,11 +190,11 @@ SELECT toTypeName(variantType(v)) FROM test LIMIT 1; └─────────────────────────────────────────────────────────────────────┘ ``` -## Conversion between Variant column and other columns +## Conversion between a Variant column and other columns -There are 4 possible conversions that can be performed with Variant column. +There are 4 possible conversions that can be performed with a column of type `Variant`. -### Converting String column to a Variant column +### Converting a String column to a Variant column Conversion from `String` to `Variant` is performed by parsing a value of `Variant` type from the string value: @@ -230,7 +230,7 @@ SELECT CAST(map('key1', '42', 'key2', 'true', 'key3', '2020-01-01'), 'Map(String ### Converting an ordinary column to a Variant column -It is possible to convert ordinary column with type `T` to a `Variant` column containing this type: +It is possible to convert an ordinary column with type `T` to a `Variant` column containing this type: ```sql SELECT toTypeName(variant) as type_name, [1,2,3]::Array(UInt64)::Variant(UInt64, String, Array(UInt64)) as variant, variantType(variant) as variant_name From 5f954e71254640219248169f8eecdbb8bf79c246 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Mar 2024 21:20:34 +0000 Subject: [PATCH 0719/1165] Fix test in aarch64 build --- ...e_nulls_injective_functions_and_analyzer.reference | 11 +++++++---- ..._by_use_nulls_injective_functions_and_analyzer.sql | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.reference b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.reference index 13248cf394f..e90d21df61f 100644 --- a/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.reference +++ b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.reference @@ -1,10 +1,13 @@ +3 4 5 +6 7 -19 -259 -65539 -4294967299 +8 +9 +10 +11 +12 \N (((0))) (((0))) diff --git a/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql index 003566014a0..e7ea964b876 100644 --- a/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql +++ b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql @@ -1,5 +1,5 @@ set allow_experimental_analyzer=1, group_by_use_nulls=1, optimize_injective_functions_in_group_by=1; -SELECT intExp2(intExp2(number)) + 3 FROM numbers(10) GROUP BY GROUPING SETS (('str', intExp2(intExp2(number))), ('str')) order by all; +SELECT bitNot(bitNot(number)) + 3 FROM numbers(10) GROUP BY GROUPING SETS (('str', bitNot(bitNot(number))), ('str')) order by all; SELECT tuple(tuple(tuple(number))) FROM numbers(10) GROUP BY GROUPING SETS (('str', tuple(tuple(number))), ('str')) order by all; SELECT materialize(3) + 3 FROM numbers(10) GROUP BY GROUPING SETS (('str', materialize(materialize(3))), ('str')) order by all; From ffd29f8b72f22f6728862128cfdface206905e83 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 27 Mar 2024 22:29:36 +0100 Subject: [PATCH 0720/1165] Update test.py --- tests/integration/test_broken_projections/test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py index adb7b58f171..ccc64de4133 100644 --- a/tests/integration/test_broken_projections/test.py +++ b/tests/integration/test_broken_projections/test.py @@ -736,7 +736,10 @@ def test_mutation_with_broken_projection(cluster): "all_2_2_0_5" in broken or "" == broken ) # second could be because of a merge. - check(node, table_name, 0) + if "" == broken: + check(node, table_name, 1) + else: + check(node, table_name, 0) node.query( f"ALTER TABLE {table_name} DELETE WHERE c == 13 SETTINGS mutations_sync = 1" From 90144f7ae4d7c4238b1578a365cd2f6b1b44cb8f Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Mar 2024 21:31:15 +0000 Subject: [PATCH 0721/1165] Fix style --- src/Functions/FunctionsConversion.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 6a1a2d5e888..448a5fd8fc6 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -89,7 +89,6 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NOT_IMPLEMENTED; extern const int CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN; - extern const int CANNOT_PARSE_BOOL; extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; } From 15566f68c11c938407ef8c8982cd312573bf0a73 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 27 Mar 2024 22:58:48 +0100 Subject: [PATCH 0722/1165] fix test --- tests/queries/0_stateless/01271_show_privileges.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index b18ae8a99be..b1237189cb3 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -155,7 +155,7 @@ SYSTEM FLUSH ASYNC INSERT QUEUE ['FLUSH ASYNC INSERT QUEUE'] GLOBAL SYSTEM FLUSH SYSTEM FLUSH [] \N SYSTEM SYSTEM THREAD FUZZER ['SYSTEM START THREAD FUZZER','SYSTEM STOP THREAD FUZZER','START THREAD FUZZER','STOP THREAD FUZZER'] GLOBAL SYSTEM SYSTEM UNFREEZE ['SYSTEM UNFREEZE'] GLOBAL SYSTEM -SYSTEM FAILPOINT ['SYSTEM ENABLE FAILPOINT','SYSTEM DISABLE FAILPOINT'] GLOBAL SYSTEM +SYSTEM FAILPOINT ['SYSTEM ENABLE FAILPOINT','SYSTEM DISABLE FAILPOINT','SYSTEM WAIT FAILPOINT'] GLOBAL SYSTEM SYSTEM LISTEN ['SYSTEM START LISTEN','SYSTEM STOP LISTEN'] GLOBAL SYSTEM SYSTEM JEMALLOC ['SYSTEM JEMALLOC PURGE','SYSTEM JEMALLOC ENABLE PROFILE','SYSTEM JEMALLOC DISABLE PROFILE','SYSTEM JEMALLOC FLUSH PROFILE'] GLOBAL SYSTEM SYSTEM [] \N ALL From c39cd75dff6b31c05dc179636a7d6d06e1903bf4 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Mar 2024 22:15:29 +0000 Subject: [PATCH 0723/1165] Support Variant in JSONExtract functions --- src/Functions/FunctionsJSON.h | 44 ++++++++++++++++++- .../03034_json_extract_variant.reference | 5 +++ .../03034_json_extract_variant.sql | 6 +++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03034_json_extract_variant.reference create mode 100644 tests/queries/0_stateless/03034_json_extract_variant.sql diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 8a193785f87..af7cdeccba4 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -10,7 +10,7 @@ #include #include -#include +//#include #include #include @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,8 @@ #include #include #include +#include +#include #include #include @@ -1234,6 +1237,35 @@ struct JSONExtractTree std::unique_ptr value; }; + class VariantNode : public Node + { + public: + VariantNode(std::vector> variant_nodes_, std::vector order_) : variant_nodes(std::move(variant_nodes_)), order(std::move(order_)) { } + + bool insertResultToColumn(IColumn & dest, const Element & element) override + { + auto & column_variant = assert_cast(dest); + for (size_t i : order) + { + auto & variant = column_variant.getVariantByGlobalDiscriminator(i); + if (variant_nodes[i]->insertResultToColumn(variant, element)) + { + column_variant.getLocalDiscriminators().push_back(column_variant.localDiscriminatorByGlobal(i)); + column_variant.getOffsets().push_back(variant.size() - 1); + return true; + } + } + + return false; + } + + private: + std::vector> variant_nodes; + /// Order in which we should try variants nodes. + /// For example, String should be always the last one. + std::vector order; + }; + static std::unique_ptr build(const char * function_name, const DataTypePtr & type) { switch (type->getTypeId()) @@ -1310,6 +1342,16 @@ struct JSONExtractTree const auto & value_type = map_type.getValueType(); return std::make_unique(build(function_name, key_type), build(function_name, value_type)); } + case TypeIndex::Variant: + { + const auto & variant_type = static_cast(*type); + const auto & variants = variant_type.getVariants(); + std::vector> variant_nodes; + variant_nodes.reserve(variants.size()); + for (const auto & variant : variants) + variant_nodes.push_back(build(function_name, variant)); + return std::make_unique(std::move(variant_nodes), SerializationVariant::getVariantsDeserializeTextOrder(variants)); + } default: throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} doesn't support the return type schema: {}", diff --git a/tests/queries/0_stateless/03034_json_extract_variant.reference b/tests/queries/0_stateless/03034_json_extract_variant.reference new file mode 100644 index 00000000000..8d78d639fd2 --- /dev/null +++ b/tests/queries/0_stateless/03034_json_extract_variant.reference @@ -0,0 +1,5 @@ +42 UInt32 +Hello String +[1,2,3] Array(UInt32) +{'a':42,'b':'Hello','c':[1,2,3]} +[('a',42),('b','Hello'),('c',[1,2,3])] Array(Tuple(String, Variant(Array(UInt32), String, UInt32))) diff --git a/tests/queries/0_stateless/03034_json_extract_variant.sql b/tests/queries/0_stateless/03034_json_extract_variant.sql new file mode 100644 index 00000000000..54d5bed9582 --- /dev/null +++ b/tests/queries/0_stateless/03034_json_extract_variant.sql @@ -0,0 +1,6 @@ +select JSONExtract('{"a" : 42}', 'a', 'Variant(String, UInt32)') as v, variantType(v); +select JSONExtract('{"a" : "Hello"}', 'a', 'Variant(String, UInt32)') as v, variantType(v); +select JSONExtract('{"a" : [1, 2, 3]}', 'a', 'Variant(String, Array(UInt32))') as v, variantType(v); +select JSONExtract('{"obj" : {"a" : 42, "b" : "Hello", "c" : [1,2,3]}}', 'obj', 'Map(String, Variant(UInt32, String, Array(UInt32)))'); +select JSONExtractKeysAndValues('{"a" : 42, "b" : "Hello", "c" : [1,2,3]}', 'Variant(UInt32, String, Array(UInt32))') as v, toTypeName(v); + From 0772536a4c6addf790fca729611feeb430a0d63a Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Mar 2024 22:26:44 +0000 Subject: [PATCH 0724/1165] Add examples in docs --- docs/en/sql-reference/data-types/variant.md | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/en/sql-reference/data-types/variant.md b/docs/en/sql-reference/data-types/variant.md index 7d10d4b0e97..668a1b260f4 100644 --- a/docs/en/sql-reference/data-types/variant.md +++ b/docs/en/sql-reference/data-types/variant.md @@ -395,3 +395,37 @@ SELECT v, variantType(v) FROM test ORDER by v; │ 100 │ UInt32 │ └─────┴────────────────┘ ``` + +## JSONExtract functions with Variant + +All `JSONExtract*` functions support `Variant` type: + +```sql +SELECT JSONExtract('{"a" : [1, 2, 3]}', 'a', 'Variant(UInt32, String, Array(UInt32))') AS variant, variantType(variant) AS variant_type; +``` + +```text +┌─variant─┬─variant_type──┐ +│ [1,2,3] │ Array(UInt32) │ +└─────────┴───────────────┘ +``` + +```sql +SELECT JSONExtract('{"obj" : {"a" : 42, "b" : "Hello", "c" : [1,2,3]}}', 'obj', 'Map(String, Variant(UInt32, String, Array(UInt32)))') AS map_of_variants, mapApply((k, v) -> (k, variantType(v)), map_of_variants) AS map_of_variant_types +``` + +```text +┌─map_of_variants──────────────────┬─map_of_variant_types────────────────────────────┐ +│ {'a':42,'b':'Hello','c':[1,2,3]} │ {'a':'UInt32','b':'String','c':'Array(UInt32)'} │ +└──────────────────────────────────┴─────────────────────────────────────────────────┘ +``` + +```sql +SELECT JSONExtractKeysAndValues('{"a" : 42, "b" : "Hello", "c" : [1,2,3]}', 'Variant(UInt32, String, Array(UInt32))') AS variants, arrayMap(x -> (x.1, variantType(x.2)), variants) AS variant_types +``` + +```text +┌─variants───────────────────────────────┬─variant_types─────────────────────────────────────────┐ +│ [('a',42),('b','Hello'),('c',[1,2,3])] │ [('a','UInt32'),('b','String'),('c','Array(UInt32)')] │ +└────────────────────────────────────────┴───────────────────────────────────────────────────────┘ +``` From 75be74874e15edf718b205f5ee2836ce7c5e54b6 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Mar 2024 22:27:27 +0000 Subject: [PATCH 0725/1165] Fix include --- src/Functions/FunctionsJSON.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index af7cdeccba4..33dd40aed90 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -10,7 +10,7 @@ #include #include -//#include +#include #include #include From 7ab77758a559511144464092853e01fcb7a56bdc Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Wed, 27 Mar 2024 16:43:36 -0700 Subject: [PATCH 0726/1165] [Docs] Add info on JSON formatting for logger --- .../settings.md | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 5d87ca5ce8d..436321c8fe8 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -1354,6 +1354,7 @@ Keys: - `count` – The number of archived log files that ClickHouse stores. - `console` – Send `log` and `errorlog` to the console instead of file. To enable, set to `1` or `true`. - `stream_compress` – Compress `log` and `errorlog` with `lz4` stream compression. To enable, set to `1` or `true`. +- `formatting` – Specify log format to be printed in console log (currently only `json` supported). Both log and error log file names (only file names, not directories) support date and time format specifiers. @@ -1422,6 +1423,8 @@ Writing to the console can be configured. Config example: ``` +### syslog + Writing to the syslog is also supported. Config example: ``` xml @@ -1445,6 +1448,52 @@ Keys for syslog: Default value: `LOG_USER` if `address` is specified, `LOG_DAEMON` otherwise. - format – Message format. Possible values: `bsd` and `syslog.` +### Log formats + +You can specify the log format that will be outputted in the console log. Currently, only JSON is supported. Here is an example of an output JSON log: + +```json +{ + "date_time": "1650918987.180175", + "thread_name": "#1", + "thread_id": "254545", + "level": "Trace", + "query_id": "", + "logger_name": "BaseDaemon", + "message": "Received signal 2", + "source_file": "../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()", + "source_line": "192" +} +``` +To enable JSON logging support, use the following snippet: + +```xml + + + json + + date_time + thread_name + thread_id + level + query_id + logger_name + message + source_file + source_line + + + +``` + +**Renaming keys for JSON logs** + +Key names can be modified by changing tag values inside the `` tag. For example, to change `DATE_TIME` to `MY_DATE_TIME`, you can use `MY_DATE_TIME`. + +**Omitting keys for JSON logs** + +Log properties can be omitted by commenting out the property. For example, if you do not want your log to print `query_id`, you can comment out the `` tag. + ## send_crash_reports {#send_crash_reports} Settings for opt-in sending crash reports to the ClickHouse core developers team via [Sentry](https://sentry.io). From 72f3871bd364a24d4a0b58d8a8078ebb069dd0be Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Thu, 28 Mar 2024 01:58:17 +0100 Subject: [PATCH 0727/1165] Fix clickhouse-test --- tests/clickhouse-test | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 10851d23481..2f1a59dceb6 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -451,8 +451,7 @@ def get_processlist_size(args): """ SELECT count() - FROM - FROM system.processes + FROM clusterAllReplicas('test_cluster_database_replicated', system.processes) WHERE query NOT LIKE '%system.processes%' """, ).strip() From 54348e5e8c0636ae731b434a641cf7d1fc0f80e0 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Thu, 28 Mar 2024 06:27:48 +0100 Subject: [PATCH 0728/1165] Fix logs saving in DatabaseReplicated tests --- docker/test/stateless/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index bac9d8df7a9..c6b3a9a99f0 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -257,10 +257,10 @@ do echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then - err=$( { clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) + err=$( { clickhouse-client -q --port 19000 "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 - err=$( { clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 ) + err=$( { clickhouse-client -q --port 29000 "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 ) echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 fi From dae3a9e3896c0da5df4331776efdff6346c390f4 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Thu, 28 Mar 2024 06:31:46 +0100 Subject: [PATCH 0729/1165] Fix --- docker/test/stateless/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index c6b3a9a99f0..b9ed0561a48 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -257,10 +257,10 @@ do echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then - err=$( { clickhouse-client -q --port 19000 "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) + err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 - err=$( { clickhouse-client -q --port 29000 "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 ) + err=$( { clickhouse-client --port 29000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 ) echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 fi From afbb3f8a506b29ef085b67d936cb2dc1321fa20f Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 28 Mar 2024 09:26:00 +0100 Subject: [PATCH 0730/1165] Added wait before commit blocks --- src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index 9432cdf9fef..fe64415191c 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -150,6 +150,7 @@ void WriteBufferFromAzureBlobStorage::finalizeImpl() if (!block_ids.empty()) { + task_tracker->waitAll(); auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path); execWithRetry([&](){ block_blob_client.CommitBlockList(block_ids); }, max_unexpected_write_error_retries); LOG_TRACE(log, "Committed {} blocks for blob `{}`", block_ids.size(), blob_path); From 5a0f78ed6a565d140d94a2b64c059eb4cb033835 Mon Sep 17 00:00:00 2001 From: justindeguzman Date: Thu, 28 Mar 2024 01:47:35 -0700 Subject: [PATCH 0731/1165] [Docs] Remove duplicate Chinese MaterializedMySQL page --- .../database-engines/materialize-mysql.md | 196 ------------------ 1 file changed, 196 deletions(-) delete mode 100644 docs/zh/engines/database-engines/materialize-mysql.md diff --git a/docs/zh/engines/database-engines/materialize-mysql.md b/docs/zh/engines/database-engines/materialize-mysql.md deleted file mode 100644 index 5d1394f9456..00000000000 --- a/docs/zh/engines/database-engines/materialize-mysql.md +++ /dev/null @@ -1,196 +0,0 @@ ---- -slug: /zh/engines/database-engines/materialize-mysql -sidebar_position: 29 -sidebar_label: "[experimental] MaterializedMySQL" ---- - -# [experimental] MaterializedMySQL {#materialized-mysql} - -**这是一个实验性的特性，不应该在生产中使用。** - -创建ClickHouse数据库，包含MySQL中所有的表，以及这些表中的所有数据。 - -ClickHouse服务器作为MySQL副本工作。它读取binlog并执行DDL和DML查询。 - -这个功能是实验性的。 - -## 创建数据库 {#creating-a-database} - -``` sql -CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MaterializeMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...] -``` - -**引擎参数** - -- `host:port` — MySQL服务地址 -- `database` — MySQL数据库名称 -- `user` — MySQL用户名 -- `password` — MySQL用户密码 - -**引擎配置** - -- `max_rows_in_buffer` — 允许数据缓存到内存中的最大行数(对于单个表和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `65505`。 -- `max_bytes_in_buffer` — 允许在内存中缓存数据的最大字节数(对于单个表和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `1048576`. -- `max_rows_in_buffers` — 允许数据缓存到内存中的最大行数(对于数据库和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `65505`. -- `max_bytes_in_buffers` — 允许在内存中缓存数据的最大字节数(对于数据库和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `1048576`. -- `max_flush_data_time` — 允许数据在内存中缓存的最大毫秒数(对于数据库和无法查询的缓存数据)。当超过这个时间时，数据将被物化。默认值: `1000`. -- `max_wait_time_when_mysql_unavailable` — 当MySQL不可用时重试间隔(毫秒)。负值禁止重试。默认值: `1000`. -- `allows_query_when_mysql_lost` — 当mysql丢失时，允许查询物化表。默认值: `0` (`false`). -``` -CREATE DATABASE mysql ENGINE = MaterializeMySQL('localhost:3306', 'db', 'user', '***') - SETTINGS - allows_query_when_mysql_lost=true, - max_wait_time_when_mysql_unavailable=10000; -``` - -**MySQL服务器端配置** - -为了`MaterializeMySQL`正确的工作，有一些强制性的`MySQL`侧配置设置应该设置: - -- `default_authentication_plugin = mysql_native_password`，因为`MaterializeMySQL`只能使用此方法授权。 -- `gtid_mode = on`，因为要提供正确的`MaterializeMySQL`复制，基于GTID的日志记录是必须的。注意，在打开这个模式`On`时，你还应该指定`enforce_gtid_consistency = on`。 - -## 虚拟列 {#virtual-columns} - -当使用`MaterializeMySQL`数据库引擎时，[ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md)表与虚拟的`_sign`和`_version`列一起使用。 - -- `_version` — 同步版本。 类型[UInt64](../../sql-reference/data-types/int-uint.md). -- `_sign` — 删除标记。类型 [Int8](../../sql-reference/data-types/int-uint.md). Possible values: - - `1` — 行不会删除, - - `-1` — 行被删除。 - -## 支持的数据类型 {#data_types-support} - -| MySQL | ClickHouse | -|-------------------------|--------------------------------------------------------------| -| TINY | [Int8](../../sql-reference/data-types/int-uint.md) | -| SHORT | [Int16](../../sql-reference/data-types/int-uint.md) | -| INT24 | [Int32](../../sql-reference/data-types/int-uint.md) | -| LONG | [UInt32](../../sql-reference/data-types/int-uint.md) | -| LONGLONG | [UInt64](../../sql-reference/data-types/int-uint.md) | -| FLOAT | [Float32](../../sql-reference/data-types/float.md) | -| DOUBLE | [Float64](../../sql-reference/data-types/float.md) | -| DECIMAL, NEWDECIMAL | [Decimal](../../sql-reference/data-types/decimal.md) | -| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) | -| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) | -| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) | -| ENUM | [Enum](../../sql-reference/data-types/enum.md) | -| STRING | [String](../../sql-reference/data-types/string.md) | -| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) | -| BLOB | [String](../../sql-reference/data-types/string.md) | -| BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) | - -不支持其他类型。如果MySQL表包含此类类型的列，ClickHouse抛出异常"Unhandled data type"并停止复制。 - -[Nullable](../../sql-reference/data-types/nullable.md)已经支持 - -## 使用方式 {#specifics-and-recommendations} - -### 兼容性限制 - -除了数据类型的限制外，与`MySQL`数据库相比，还存在一些限制，在实现复制之前应先解决这些限制： - -- `MySQL`中的每个表都应该包含`PRIMARY KEY` - -- 对于包含`ENUM`字段值超出范围（在`ENUM`签名中指定）的行的表，复制将不起作用。 - -### DDL查询 {#ddl-queries} - -MySQL DDL查询转换为相应的ClickHouse DDL查询([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md))。如果ClickHouse无法解析某个DDL查询，则该查询将被忽略。 - -### Data Replication {#data-replication} - -`MaterializeMySQL`不支持直接`INSERT`, `DELETE`和`UPDATE`查询. 但是，它们是在数据复制方面支持的： - -- MySQL的`INSERT`查询转换为`INSERT`并携带`_sign=1`. - -- MySQL的`DELETE`查询转换为`INSERT`并携带`_sign=-1`. - -- MySQL的`UPDATE`查询转换为`INSERT`并携带`_sign=-1`, `INSERT`和`_sign=1`. - -### 查询MaterializeMySQL表 {#select} - -`SELECT`查询`MaterializeMySQL`表有一些细节: - -- 如果`_version`在`SELECT`中没有指定，则使用[FINAL](../../sql-reference/statements/select/from.md#select-from-final)修饰符。所以只有带有`MAX(_version)`的行才会被选中。 - -- 如果`_sign`在`SELECT`中没有指定，则默认使用`WHERE _sign=1`。因此，删除的行不会包含在结果集中。 - -- 结果包括列中的列注释，因为它们存在于SQL数据库表中。 - -### Index Conversion {#index-conversion} - -MySQL的`PRIMARY KEY`和`INDEX`子句在ClickHouse表中转换为`ORDER BY`元组。 - -ClickHouse只有一个物理顺序，由`ORDER BY`子句决定。要创建一个新的物理顺序，使用[materialized views](../../sql-reference/statements/create/view.md#materialized)。 - -**Notes** - -- 带有`_sign=-1`的行不会从表中物理删除。 -- `MaterializeMySQL`引擎不支持级联`UPDATE/DELETE`查询。 -- 复制很容易被破坏。 -- 禁止对数据库和表进行手工操作。 -- `MaterializeMySQL`受[optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert)设置的影响。当MySQL服务器中的表发生变化时，数据会合并到`MaterializeMySQL`数据库中相应的表中。 - -## 使用示例 {#examples-of-use} - -MySQL操作: - -``` sql -mysql> CREATE DATABASE db; -mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT); -mysql> INSERT INTO db.test VALUES (1, 11), (2, 22); -mysql> DELETE FROM db.test WHERE a=1; -mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16); -mysql> UPDATE db.test SET c='Wow!', b=222; -mysql> SELECT * FROM test; -``` - -```text -+---+------+------+ -| a | b | c | -+---+------+------+ -| 2 | 222 | Wow! | -+---+------+------+ -``` - -ClickHouse中的数据库，与MySQL服务器交换数据: - -创建的数据库和表: - -``` sql -CREATE DATABASE mysql ENGINE = MaterializeMySQL('localhost:3306', 'db', 'user', '***'); -SHOW TABLES FROM mysql; -``` - -``` text -┌─name─┐ -│ test │ -└──────┘ -``` - -然后插入数据: - -``` sql -SELECT * FROM mysql.test; -``` - -``` text -┌─a─┬──b─┐ -│ 1 │ 11 │ -│ 2 │ 22 │ -└───┴────┘ -``` - -删除数据后，添加列并更新: - -``` sql -SELECT * FROM mysql.test; -``` - -``` text -┌─a─┬───b─┬─c────┐ -│ 2 │ 222 │ Wow! │ -└───┴─────┴──────┘ -``` From 241e68c44b234785aa783dd0888d15a578a3a5ed Mon Sep 17 00:00:00 2001 From: Blargian Date: Thu, 28 Mar 2024 11:22:28 +0100 Subject: [PATCH 0732/1165] Add documentation for hasToken functions --- .../functions/other-functions.md | 48 +++++- .../functions/string-search-functions.md | 151 ++++++++++++++++++ 2 files changed, 196 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index e7fca31483a..2638a5a650b 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -543,12 +543,54 @@ You can get similar result by using the [ternary operator](../../sql-reference/f Returns 1 if the Float32 and Float64 argument is NaN, otherwise this function 0. -## hasColumnInTable(\[‘hostname’\[, ‘username’\[, ‘password’\]\],\] ‘database’, ‘table’, ‘column’) +## hasColumnInTable + +Given the database name, the table name, and the column name as constant strings, returns 1 if the given column exists, otherwise 0. + +**Syntax** + +```sql +hasColumnInTable(\[‘hostname’\[, ‘username’\[, ‘password’\]\],\] ‘database’, ‘table’, ‘column’) +``` + +**Parameters** + +- `database` : name of the database. [String literal](../syntax#syntax-string-literal) +- `table` : name of the table. [String literal](../syntax#syntax-string-literal) +- `column` : name of the column. [String literal](../syntax#syntax-string-literal) +- `hostname` : remote server name to perform the check on. [String literal](../syntax#syntax-string-literal) +- `username` : username for remote server. [String literal](../syntax#syntax-string-literal) +- `password` : password for remote server. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- `1` if the given column exists. +- `0`, otherwise. + +**Implementation details** -Given the database name, the table name, and the column name as constant strings, returns 1 if the given column exists, otherwise 0. If parameter `hostname` is given, the check is performed on a remote server. -If the table does not exist, an exception is thrown. For elements in a nested data structure, the function checks for the existence of a column. For the nested data structure itself, the function returns 0. +**Example** + +Query: + +```sql +SELECT hasColumnInTable('system','metrics','metric') +``` + +```response +1 +``` + +```sql +SELECT hasColumnInTable('system','metrics','non-existing_column') +``` + +```response +0 +``` + ## bar Builds a bar chart. diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 22f879c62ae..818626fc4fc 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -685,3 +685,154 @@ Like [hasSubsequence](#hasSubsequence) but assumes `haystack` and `needle` are U ## hasSubsequenceCaseInsensitiveUTF8 Like [hasSubsequenceUTF8](#hasSubsequenceUTF8) but searches case-insensitively. + +## hasToken + +Returns 1 if a given token is present in a haystack, or 0 otherwise. + +**Syntax** + +```sql +hasToken(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, if the token is not present. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + +Query: + +```sql +SELECT hasToken('Hello World','Hello'); +``` + +```response +1 +``` + +## hasTokenOrNull + +Returns 1 if a given token is present, 0 if not present, and null if the token is ill-formed. + +**Syntax** + +```sql +hasTokenOrNull(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, if the token is not present in the haystack. +- null, if the token is ill-formed. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + +Where `hasToken` would throw an error for an ill-formed token, `hasTokenOrNull` returns `null` for an ill-formed token. + +Query: + +```sql +SELECT hasTokenOrNull('Hello World','Hello,World'); +``` + +```response +null +``` + +## hasTokenCaseInsensitive + +Returns 1 if a given token is present in a haystack, 0 otherwise. Ignores case. + +**Syntax** + +```sql +hasTokenCaseInsensitive(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, otherwise. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + +Query: + +```sql +SELECT hasTokenCaseInsensitive('Hello World','hello'); +``` + +```response +1 +``` + +## hasTokenCaseInsensitiveOrNull + +Returns 1 if a given token is present in a haystack, 0 otherwise. Ignores case and returns null if the token is ill-formed. + +**Syntax** + +```sql +hasTokenCaseInsensitive(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, if token is not present. +- null, if the token is ill-formed. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + + +Where `hasTokenCaseInsensitive` would throw an error for an ill-formed token, `hasTokenCaseInsensitiveOrNull` returns `null` for an ill-formed token. + +Query: + +```sql +SELECT hasTokenCaseInsensitiveOrNull('Hello World','hello,world'); +``` + +```response +null +``` \ No newline at end of file From e2bc301b0073af2916322e02b067bc26d319e45d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Mar 2024 12:02:32 +0100 Subject: [PATCH 0733/1165] Update analyzer_tech_debt.txt --- tests/analyzer_tech_debt.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt index 89af93f581b..5f798158a41 100644 --- a/tests/analyzer_tech_debt.txt +++ b/tests/analyzer_tech_debt.txt @@ -6,5 +6,6 @@ # Flaky list 01825_type_json_in_array 01414_mutations_and_errors_zookeeper +01287_max_execution_speed # Check after ConstantNode refactoring 02154_parser_backtracking From f5437d3055835b06b3f5a800944581d9b9cdeb2c Mon Sep 17 00:00:00 2001 From: KrJin <33444112+jincong8973@users.noreply.github.com> Date: Thu, 28 Mar 2024 19:08:38 +0800 Subject: [PATCH 0734/1165] Update string-search-functions.md Improve zh-cn doc of string-search-functions --- .../functions/string-search-functions.md | 698 ++++++++++++++++-- 1 file changed, 636 insertions(+), 62 deletions(-) diff --git a/docs/zh/sql-reference/functions/string-search-functions.md b/docs/zh/sql-reference/functions/string-search-functions.md index e4167127424..14c40fa1243 100644 --- a/docs/zh/sql-reference/functions/string-search-functions.md +++ b/docs/zh/sql-reference/functions/string-search-functions.md @@ -1,128 +1,702 @@ --- slug: /zh/sql-reference/functions/string-search-functions --- -# 字符串搜索函数 {#zi-fu-chuan-sou-suo-han-shu} -下列所有函数在默认的情况下区分大小写。对于不区分大小写的搜索，存在单独的变体。 +# 字符串搜索函数 -## 位置（大海捞针），定位（大海捞针) {#positionhaystack-needle-locatehaystack-needle} +本节中的所有函数默认情况下都区分大小写进行搜索。不区分大小写的搜索通常由单独的函数变体提供。 +请注意，不区分大小写的搜索，遵循英语的小写-大写规则。 +例如。英语中大写的`i`是`I`，而在土耳其语中则是`İ`, 对于英语以外的语言，结果可能会不符合预期。 -在字符串`haystack`中搜索子串`needle`。 -返回子串的位置（以字节为单位），从1开始，如果未找到子串，则返回0。 +本节中的函数还假设搜索字符串和被搜索字符串是单字节编码文本(例如ASCII)。如果违反此假设，不会抛出异常且结果为undefined。 +UTF-8 编码字符串的搜索通常由单独的函数变体提供。同样，如果使用 UTF-8 函数变体但输入字符串不是 UTF-8 编码文本，不会抛出异常且结果为undefined。 +需要注意，函数不会执行自动 Unicode 规范化，您可以使用[normalizeUTF8*()](https://clickhouse.com/docs/zh/sql-reference/functions/string-functions/) 函数来执行此操作。 +在[字符串函数](string-functions.md) 和 [字符串替换函数](string-replace-functions.md) 会分别说明. -对于不区分大小写的搜索，请使用函数`positionCaseInsensitive`。 +## position -## positionUTF8(大海捞针) {#positionutf8haystack-needle} +返回字符串`haystack`中子字符串`needle`的位置（以字节为单位，从 1 开始）。 -与`position`相同，但位置以Unicode字符返回。此函数工作在UTF-8编码的文本字符集中。如非此编码的字符集，则返回一些非预期结果（他不会抛出异常）。 +**语法** -对于不区分大小写的搜索，请使用函数`positionCaseInsensitiveUTF8`。 +``` sql +position(haystack, needle[, start_pos]) +``` -## 多搜索分配（干草堆，\[针1，针2, …, needlen\]) {#multisearchallpositionshaystack-needle1-needle2-needlen} +别名： +- `position(needle IN haystack)` -与`position`相同，但函数返回一个数组，其中包含所有匹配needle我的位置。 +**参数** -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchAllPositionsCaseInsensitive，multiSearchAllPositionsUTF8，multiSearchAllPositionsCaseInsensitiveUTF8`。 +- `haystack` — 被检索查询字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` – 在字符串`haystack` 中开始检索的位置(从1开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选 -## multiSearchFirstPosition(大海捞针,\[针1，针2, …, needlen\]) {#multisearchfirstpositionhaystack-needle1-needle2-needlen} +**返回值** -与`position`相同，但返回在`haystack`中与needles字符串匹配的最左偏移。 +- 若子字符串存在，返回位置(以字节为单位，从 1 开始）。 +- 如果不存在子字符串,返回0。 -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchFirstPositionCaseInsensitive，multiSearchFirstPositionUTF8，multiSearchFirstPositionCaseInsensitiveUTF8`。 +如果子字符串 `needle` 为空，则： +- 如果未指定 `start_pos`，返回 `1` +- 如果 `start_pos` 为 0，则返回 `1` +- 如果 `start_pos >= 1` 且 `start_pos <= length(haystack) + 1`，则返回 `start_pos` +- 否则返回 `0` -## multiSearchFirstIndex(大海捞针,\[针1，针2, …, needlen\]) {#multisearchfirstindexhaystack-needle1-needle2-needlen} +以上规则同样在这些函数中生效: [locate](#locate), [positionCaseInsensitive](#positionCaseInsensitive), [positionUTF8](#positionUTF8), [positionCaseInsensitiveUTF8](#positionCaseInsensitiveUTF8) -返回在字符串`haystack`中最先查找到的needle我的索引`i`（从1开始），没有找到任何匹配项则返回0。 +数据类型: `Integer`. -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchFirstIndexCaseInsensitive，multiSearchFirstIndexUTF8，multiSearchFirstIndexCaseInsensitiveUTF8`。 +**示例** -## 多搜索（大海捞针，\[针1，针2, …, needlen\]) {#multisearchanyhaystack-needle1-needle2-needlen} +``` sql +SELECT position('Hello, world!', '!'); +``` -如果`haystack`中至少存在一个needle我匹配则返回1，否则返回0。 +结果： + +``` text +┌─position('Hello, world!', '!')─┐ +│ 13 │ +└────────────────────────────────┘ +``` + +示例，使用参数 `start_pos` : + +``` sql +SELECT + position('Hello, world!', 'o', 1), + position('Hello, world!', 'o', 7) +``` +结果： +``` text +┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐ +│ 5 │ 9 │ +└───────────────────────────────────┴───────────────────────────────────┘ +``` + +示例，语法别名 `needle IN haystack`: + +```sql +SELECT 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); +``` + +结果： + +```text +┌─equals(6, position(s, '/'))─┐ +│ 1 │ +└─────────────────────────────┘ +``` + +示例，子字符串 `needle` 为空: + +``` sql +SELECT + position('abc', ''), + position('abc', '', 0), + position('abc', '', 1), + position('abc', '', 2), + position('abc', '', 3), + position('abc', '', 4), + position('abc', '', 5) +``` +结果： +``` text +┌─position('abc', '')─┬─position('abc', '', 0)─┬─position('abc', '', 1)─┬─position('abc', '', 2)─┬─position('abc', '', 3)─┬─position('abc', '', 4)─┬─position('abc', '', 5)─┐ +│ 1 │ 1 │ 1 │ 2 │ 3 │ 4 │ 0 │ +└─────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┘ +``` + +## locate + +类似于 [position](#position) but with arguments `haystack` and `locate` switched. + +此函数的行为取决于 ClickHouse 版本： +- in versions < v24.3, `locate` was an alias of function `position` and accepted arguments `(haystack, needle[, start_pos])`. +- 在 v24.3 以下的版本中，`locate` 是函数`position`的别名，参数为 `(haystack, needle[, start_pos])`。 +- 在 v24.3 及以上的版本中,, `locate` 是独立的函数 (以更好地兼容 MySQL) ,参数为 `(needle, haystack[, start_pos])`。 之前的行为 + 可以在设置中恢复 [function_locate_has_mysql_compatible_argument_order = false](../../operations/settings/settings.md#function-locate-has-mysql-compatible-argument-order); + +**语法** + +``` sql +locate(needle, haystack[, start_pos]) +``` + +## positionCaseInsensitive + +类似于 [position](#position) 但是不区分大小写。 + +## positionUTF8 + +类似于 [position](#position) 但是假定 `haystack` 和 `needle` 是 UTF-8 编码的字符串。 + +**示例** + +函数 `positionUTF8` 可以正确的将字符 `ö` 计为单个 Unicode 代码点(`ö`由两个点表示): + +``` sql +SELECT positionUTF8('Motörhead', 'r'); +``` + +结果： + +``` text +┌─position('Motörhead', 'r')─┐ +│ 5 │ +└────────────────────────────┘ +``` + +## positionCaseInsensitiveUTF8 + +类似于 [positionUTF8](#positionutf8) 但是不区分大小写。 + +## multiSearchAllPositions + +类似于 [position](#position)返回多个在字符串 `haystack` 中 `needle` 子字符串的位置的数组（以字节为单位，从 1 开始）。 -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchAnyCaseInsensitive，multiSearchAnyUTF8，multiSearchAnyCaseInsensitiveUTF8`。 :::note -在所有`multiSearch*`函数中，由于实现规范，needles的数量应小于28。 +所有以 `multiSearch*()` 开头的函数最多支持28 个`needle`. ::: -## 匹配（大海捞针，模式) {#matchhaystack-pattern} +**语法** -检查字符串是否与`pattern`正则表达式匹配。`pattern`可以是一个任意的`re2`正则表达式。 `re2`正则表达式的[语法](https://github.com/google/re2/wiki/Syntax)比Perl正则表达式的语法存在更多限制。 +``` sql +multiSearchAllPositions(haystack, [needle1, needle2, ..., needleN]) +``` -如果不匹配返回0，否则返回1。 +**参数** -请注意，反斜杠符号（`\`）用于在正则表达式中转义。由于字符串中采用相同的符号来进行转义。因此，为了在正则表达式中转义符号，必须在字符串文字中写入两个反斜杠（\\）。 +- `haystack` — 被检索查询字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 子字符串数组, 类型为[Array](../../sql-reference/data-types/array.md) -正则表达式与字符串一起使用，就像它是一组字节一样。正则表达式中不能包含空字节。 -对于在字符串中搜索子字符串的模式，最好使用LIKE或«position»，因为它们更加高效。 +**返回值** -## multiMatchAny（大海捞针，\[模式1，模式2, …, patternn\]) {#multimatchanyhaystack-pattern1-pattern2-patternn} +- 位置数组，数组中的每个元素对应于 `needle` 数组中的一个元素。如果在 `haystack` 中找到子字符串，则返回的数组中的元素为子字符串的位置（以字节为单位，从 1 开始）；如果未找到子字符串，则返回的数组中的元素为 0。 -与`match`相同，但如果所有正则表达式都不匹配，则返回0；如果任何模式匹配，则返回1。它使用[超扫描](https://github.com/intel/hyperscan)库。对于在字符串中搜索子字符串的模式，最好使用«multisearchany»，因为它更高效。 +**示例** + +``` sql +SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']); +``` + +结果： + +``` text +┌─multiSearchAllPositions('Hello, World!', ['hello', '!', 'world'])─┐ +│ [0,13,0] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +## multiSearchAllPositionsUTF8 + +类似于 [multiSearchAllPositions](#multiSearchAllPositions) ,但假定 `haystack` 和 `needle`-s 是 UTF-8 编码的字符串。 + +## multiSearchFirstPosition + +类似于 `position` , 在字符串`haystack`中匹配多个`needle`子字符串，从左开始任一匹配的子串，返回其位置。 + +函数 `multiSearchFirstPositionCaseInsensitive`, `multiSearchFirstPositionUTF8` and `multiSearchFirstPositionCaseInsensitiveUTF8` 提供此函数的不区分大小写 以及/或 UTF-8 变体。 + +**语法** + +```sql +multiSearchFirstPosition(haystack, [needle1, needle2, …, needleN]) +``` + +## multiSearchFirstIndex + +在字符串`haystack`中匹配多个`needle`子字符串，从左开始任一匹配的子串，返回其索引 `i` (从1开始)，如无法匹配则返回0。 + +函数 `multiSearchFirstIndexCaseInsensitive`, `multiSearchFirstIndexUTF8` 和 `multiSearchFirstIndexCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + +**语法** + +```sql +multiSearchFirstIndex(haystack, \[needle1, needle2, …, needlen\]) +``` + +## multiSearchAny {#multisearchany} + +Returns 1, if at least one string needlei matches the string `haystack` and 0 otherwise. +至少已有一个子字符串`needle`匹配 `haystack` 时返回1，否则返回 0 。 + +函数 `multiSearchAnyCaseInsensitive`, `multiSearchAnyUTF8` 和 `multiSearchAnyCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + + +**语法** + +```sql +multiSearchAny(haystack, [needle1, needle2, …, needleN]) +``` + +## match {#match} + +返回字符串 `haystack` 是否匹配正则表达式 `pattern` （[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) + +匹配基于 UTF-8，例如`.` 匹配 Unicode 代码点 `¥`，它使用两个字节以 UTF-8 表示。T正则表达式不得包含空字节。如果 `haystack` 或`pattern`不是有效的 UTF-8，则此行为为undefined。 + +与 re2 的默认行为不同，`.` 会匹配换行符。要禁用此功能，请在模式前面添加`(?-s)`。 + +如果仅希望搜索子字符串，可以使用函数 [like](#like)或 [position](#position) 来替代，这些函数的性能比此函数更高。 + +**语法** + +```sql +match(haystack, pattern) +``` + +别名： `haystack REGEXP pattern operator` + +## multiMatchAny + +类似于 `match`，如果至少有一个表达式 `patterni` 匹配字符串 `haystack`，则返回1，否则返回0。 :::note -任何`haystack`字符串的长度必须小于232\字节，否则抛出异常。这种限制是因为hyperscan API而产生的。 +`multi[Fuzzy]Match*()` 函数家族使用了(Vectorscan)[https://github.com/VectorCamp/vectorscan]库. 因此，只有当 ClickHouse 编译时支持矢量扫描时，它们才会启用。 + +要关闭所有使用矢量扫描(hyperscan)的功能，请使用设置 `SET allow_hyperscan = 0;`。 + +由于Vectorscan的限制，`haystack` 字符串的长度必须小于232字节。 + +Hyperscan 通常容易受到正则表达式拒绝服务 (ReDoS) 攻击。有关更多信息，请参见 +[https://www.usenix.org/conference/usenixsecurity22/presentation/turonova](https://www.usenix.org/conference/usenixsecurity22/presentation/turonova) +[https://doi.org/10.1007/s10664-021-10033-1](https://doi.org/10.1007/s10664-021-10033-1) +[https://doi.org/10.1145/3236024.3236027](https://doi.org/10.1145/3236024.3236027) +建议用户谨慎检查提供的表达式。 + ::: -## multiMatchAnyIndex（大海捞针，\[模式1，模式2, …, patternn\]) {#multimatchanyindexhaystack-pattern1-pattern2-patternn} +如果仅希望搜索子字符串，可以使用函数 [multiSearchAny](#multisearchany) 来替代，这些函数的性能比此函数更高。 -与`multiMatchAny`相同，但返回与haystack匹配的任何内容的索引位置。 +**语法** -## multiFuzzyMatchAny(干草堆,距离,\[模式1，模式2, …, patternn\]) {#multifuzzymatchanyhaystack-distance-pattern1-pattern2-patternn} +```sql +multiMatchAny(haystack, \[pattern1, pattern2, …, patternn\]) +``` -与`multiMatchAny`相同，但如果在haystack能够查找到任何模式匹配能够在指定的[编辑距离](https://en.wikipedia.org/wiki/Edit_distance)内进行匹配，则返回1。此功能也处于实验模式，可能非常慢。有关更多信息，请参阅[hyperscan文档](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching)。 +## multiMatchAnyIndex -## multiFuzzyMatchAnyIndex(大海捞针,距离,\[模式1，模式2, …, patternn\]) {#multifuzzymatchanyindexhaystack-distance-pattern1-pattern2-patternn} +类似于 `multiMatchAny` ，返回任何子串匹配 `haystack` 的索引。 -与`multiFuzzyMatchAny`相同，但返回匹配项的匹配能容的索引位置。 +**语法** + +```sql +multiMatchAnyIndex(haystack, \[pattern1, pattern2, …, patternn\]) +``` + +## multiMatchAllIndices + +类似于 `multiMatchAny`，返回一个数组，包含所有匹配 `haystack` 的子字符串的索引。 + +**语法** + +```sql +multiMatchAllIndices(haystack, \[pattern1, pattern2, …, patternn\]) +``` + +## multiFuzzyMatchAny + +类似于 `multiMatchAny` ，如果任一 `pattern` 匹配 `haystack`，则返回1 within a constant [edit distance](https://en.wikipedia.org/wiki/Edit_distance). 该功能依赖于实验特征 [hyperscan](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching) 库，并且对于某些边缘场景可能会很慢。性能取决于编辑距离`distance`的值和使用的`partten`，但与非模糊搜索相比，它的开销总是更高的。 :::note -`multiFuzzyMatch*`函数不支持UTF-8正则表达式，由于hyperscan限制，这些表达式被按字节解析。 +由于 hyperscan 的限制，`multiFuzzyMatch*()` 函数族不支持 UTF-8 正则表达式（hyperscan以一串字节来处理）。 ::: +**语法** + +```sql +multiFuzzyMatchAny(haystack, distance, \[pattern1, pattern2, …, patternn\]) +``` + +## multiFuzzyMatchAnyIndex + +类似于 `multiFuzzyMatchAny` 返回在编辑距离内与`haystack`匹配的任何索引 + +**语法** + +```sql +multiFuzzyMatchAnyIndex(haystack, distance, \[pattern1, pattern2, …, patternn\]) +``` + +## multiFuzzyMatchAllIndices + +类似于 `multiFuzzyMatchAny` 返回在编辑距离内与`haystack`匹配的所有索引的数组。 + +**语法** + +```sql +multiFuzzyMatchAllIndices(haystack, distance, \[pattern1, pattern2, …, patternn\]) +``` + +## extract + +使用正则表达式提取字符串。如果字符串 `haystack` 不匹配正则表达式 `pattern` ，则返回空字符串。 + +对于没有子模式的正则表达式，该函数使用与整个正则表达式匹配的片段。否则，它使用与第一个子模式匹配的片段。 + +**语法** + +```sql +extract(haystack, pattern) +``` + +## extractAll + +使用正则表达式提取字符串的所有片段。如果字符串 `haystack` 不匹配正则表达式 `pattern` ，则返回空字符串。 + +返回所有匹配项组成的字符串数组。 + +子模式的行为与函数`extract`中的行为相同。 + +**语法** + +```sql +extractAll(haystack, pattern) +``` + +## extractAllGroupsHorizontal + +使用`pattern`正则表达式匹配`haystack`字符串的所有组。 + +返回一个元素为数组的数组，其中第一个数组包含与第一组匹配的所有片段，第二个数组包含与第二组匹配的所有片段，依此类推。 + +这个函数相比 [extractAllGroupsVertical](#extractallgroups-vertical)更慢。 + +**语法** + +``` sql +extractAllGroupsHorizontal(haystack, pattern) +``` + +**参数** + +- `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含group，每个group用括号括起来。 如果 `pattern` 不包含group则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). + +**返回值** + +- 数据类型: [Array](../../sql-reference/data-types/array.md). + +如果`haystack`不匹配`pattern`正则表达式，则返回一个空数组的数组。 + +**示例** + +``` sql +SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); +``` + +结果： + +``` text +┌─extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)')─┐ +│ [['abc','def','ghi'],['111','222','333']] │ +└──────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## extractAllGroupsVertical + +使用正则表达式 `pattern`匹配字符串`haystack`中的所有group。返回一个数组，其中每个数组包含每个group的匹配片段。片段按照在`haystack`中出现的顺序进行分组。 + +**语法** + +``` sql +extractAllGroupsVertical(haystack, pattern) +``` + +**参数** + +- `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含group，每个group用括号括起来。 如果 `pattern` 不包含group则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). + +**返回值** + +- 数据类型: [Array](../../sql-reference/data-types/array.md). + +如果`haystack`不匹配`pattern`正则表达式，则返回一个空数组。 + +**示例** + +``` sql +SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); +``` + +结果： + +``` text +┌─extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)')─┐ +│ [['abc','111'],['def','222'],['ghi','333']] │ +└────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## like {#like} + +返回字符串 `haystack` 是否匹配 LIKE 表达式 `pattern`。 + +一个 LIKE 表达式可以包含普通字符和以下元字符： + +- `%` 表示任意数量的任意字符（包括零个字符）。 +- `_` 表示单个任意字符。 +- `\` 用于转义文字 `%`, `_` 和 `\`。 + +匹配基于 UTF-8，例如 `_` 匹配 Unicode 代码点 `¥`，它使用两个字节以 UTF-8 表示。 + +如果 `haystack` 或 `LIKE` 表达式不是有效的 UTF-8，则行为是未定义的。 + +不会自动执行 Unicode 规范化，您可以使用[normalizeUTF8*()](https://clickhouse.com/docs/zh/sql-reference/functions/string-functions/) 函数来执行此操作。 + +匹配字面上的 `%`, `_` 和 `/`（这些是 LIKE 元字符），请在其前面加上反斜杠：`\%`, `\_` 和 `\\`。 +如果在反斜杠前使用非 `%`, `_` 或 `\` 字符，则反斜杠将失去其特殊含义（即被解释为字面值）。 +请注意，ClickHouse 要求字符串中使用反斜杠 [也需要被转义](../syntax.md#string), 因此您实际上需要编写 `\\%`、`\\_` 和 `\\\\`。 + + +对于形式为 `%needle%` 的 LIKE 表达式，函数的性能与 `position` 函数相同。 +所有其他 LIKE 表达式都会被内部转换为正则表达式，并以与函数 `match` 相似的性能执行。 + +**语法** + +```sql +like(haystack, pattern) +``` + +别名： `haystack LIKE pattern` (operator) + +## notLike {#notlike} + +类似于 `like` 但是返回相反的结果。 + +别名： `haystack NOT LIKE pattern` (operator) + +## ilike + +类似于 `like` 但是不区分大小写。 + +别名： `haystack ILIKE pattern` (operator) + +## notILike + +类似于 `ilike` 但是返回相反的结果。 + +别名： `haystack NOT ILIKE pattern` (operator) + +## ngramDistance + +计算字符串`haystack` 和子字符串`needle`的4-gram距离。 为此，它计算两个 4-gram 多重集之间的对称差异，并通过它们的基数之和对其进行标准化。返回0-1之间的Float32。返回值越小，代表字符串越相似. Throws an exception if constant `needle` or `haystack` arguments are more than 32Kb in size. If any of non-constant `haystack` or `needle` arguments is more than 32Kb in size, the distance is always 1. + +函数 `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + +**语法** + +```sql +ngramDistance(haystack, needle) +``` + +## ngramSearch + +类似于`ngramDistance`，但计算`needle`字符串和`haystack`字符串之间的非对称差异，即来自needle的n-gram数量减去由`needle`数量归一化的n-gram的公共数量n-gram。返回 0 到 1 之间的 Float32。结果越大，’needle’越有可能在’haystack’中。该函数对于模糊字符串搜索很有用。另请参阅函数’soundex’。 +函数 `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + :::note -如要关闭所有hyperscan函数的使用，请设置`SET allow_hyperscan = 0;`。 +UTF-8变体使用了3-gram距离。这些并不是完全公平的n-gram距离。我们使用2字节的哈希函数来哈希n-gram，然后计算这些哈希表之间的(非)对称差异——可能会发生冲突。在使用UTF-8大小写不敏感格式时，我们并不使用公平的tolower函数——我们将每个码点字节的第5位（从零开始）和如果字节超过一个的零字节的第一位置零——这对拉丁字母和大部分西里尔字母都有效。 ::: -## 提取（大海捞针，图案) {#extracthaystack-pattern} +**语法** -使用正则表达式截取字符串。如果’haystack’与’pattern’不匹配，则返回空字符串。如果正则表达式中不包含子模式，它将获取与整个正则表达式匹配的子串。否则，它将获取与第一个子模式匹配的子串。 +```sql +ngramSearch(haystack, needle) +``` -## extractAll（大海捞针，图案) {#extractallhaystack-pattern} +## countSubstrings -使用正则表达式提取字符串的所有片段。如果’haystack’与’pattern’正则表达式不匹配，则返回一个空字符串。否则返回所有与正则表达式匹配的字符串数组。通常，行为与’extract’函数相同（它采用第一个子模式，如果没有子模式，则采用整个表达式）。 +返回字符串 `haystack` 中子字符串 `needle` 出现的次数。 -## 像（干草堆，模式），干草堆像模式运算符 {#likehaystack-pattern-haystack-like-pattern-operator} +函数 `countSubstringsCaseInsensitive` and `countSubstringsCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 -检查字符串是否与简单正则表达式匹配。 -正则表达式可以包含的元符号有`％`和`_`。 +**语法** -`%` 表示任何字节数（包括零字符）。 +``` sql +countSubstrings(haystack, needle[, start_pos]) +``` -`_` 表示任何一个字节。 +**参数** -可以使用反斜杠（`\`）来对元符号进行转义。请参阅«match»函数说明中有关转义的说明。 +- `haystack` — 被检索查询字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` – 在字符串`haystack` 中开始检索的位置(从1开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选 -对于像`％needle％`这样的正则表达式，改函数与`position`函数一样快。 -对于其他正则表达式，函数与’match’函数相同。 +**返回值** -## 不喜欢（干草堆，模式），干草堆不喜欢模式运算符 {#notlikehaystack-pattern-haystack-not-like-pattern-operator} +- 子字符串出现的次数。 -与’like’函数返回相反的结果。 +数据类型: [UInt64](../../sql-reference/data-types/int-uint.md). -## 大海捞针) {#ngramdistancehaystack-needle} +**示例** -基于4-gram计算`haystack`和`needle`之间的距离：计算两个4-gram集合之间的对称差异，并用它们的基数和对其进行归一化。返回0到1之间的任何浮点数 – 越接近0则表示越多的字符串彼此相似。如果常量的`needle`或`haystack`超过32KB，函数将抛出异常。如果非常量的`haystack`或`needle`字符串超过32Kb，则距离始终为1。 +``` sql +SELECT countSubstrings('aaaa', 'aa'); +``` -对于不区分大小写的搜索或/和UTF-8格式，使用函数`ngramDistanceCaseInsensitive，ngramDistanceUTF8，ngramDistanceCaseInsensitiveUTF8`。 +结果： -## ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人) {#ngramsearchhaystack-needle} +``` text +┌─countSubstrings('aaaa', 'aa')─┐ +│ 2 │ +└───────────────────────────────┘ +``` -与`ngramDistance`相同，但计算`needle`和`haystack`之间的非对称差异——`needle`的n-gram减去`needle`归一化n-gram。可用于模糊字符串搜索。 +示例，使用参数 `start_pos` : -对于不区分大小写的搜索或/和UTF-8格式，使用函数`ngramSearchCaseInsensitive，ngramSearchUTF8，ngramSearchCaseInsensitiveUTF8`。 +```sql +SELECT countSubstrings('abc___abc', 'abc', 4); +``` -:::note -对于UTF-8，我们使用3-gram。所有这些都不是完全公平的n-gram距离。我们使用2字节哈希来散列n-gram，然后计算这些哈希表之间的（非）对称差异 - 可能会发生冲突。对于UTF-8不区分大小写的格式，我们不使用公平的`tolower`函数 - 我们将每个Unicode字符字节的第5位（从零开始）和字节的第一位归零 - 这适用于拉丁语，主要用于所有西里尔字母。 -::: +结果： + +``` text +┌─countSubstrings('abc___abc', 'abc', 4)─┐ +│ 1 │ +└────────────────────────────────────────┘ +``` + +## countMatches + +返回正则表达式成功匹配的次数。 + +**语法** + +``` sql +countMatches(haystack, pattern) +``` + +**参数** + +- `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含group，每个group用括号括起来。 如果 `pattern` 不包含group则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). + +**返回值** + +- 匹配次数。 + +数据类型: [UInt64](../../sql-reference/data-types/int-uint.md). + +**示例** + +``` sql +SELECT countMatches('foobar.com', 'o+'); +``` + +结果： + +``` text +┌─countMatches('foobar.com', 'o+')─┐ +│ 2 │ +└──────────────────────────────────┘ +``` + +``` sql +SELECT countMatches('aaaa', 'aa'); +``` + +结果： + +``` text +┌─countMatches('aaaa', 'aa')────┐ +│ 2 │ +└───────────────────────────────┘ +``` + +## countMatchesCaseInsensitive + +类似于 `countMatches(haystack, pattern)` 但是不区分大小写。 + +## regexpExtract + +提取匹配正则表达式模式的字符串`haystack`中的第一个字符串，并对应于正则表达式组索引。 + +**语法** + +``` sql +regexpExtract(haystack, pattern[, index]) +``` + +别名： `REGEXP_EXTRACT(haystack, pattern[, index])`. + +**参数** + +- `haystack` — 被匹配字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `pattern` — 正则表达式，必须是常量。 [String](../../sql-reference/syntax.md#syntax-string-literal). +- `index` – 一个大于等于0的整数，默认为1，它代表要提取哪个正则表达式组。 [UInt or Int](../../sql-reference/data-types/int-uint.md) 可选。 + +**返回值** + +`pattern`可以包含多个正则组, `index` 代表要提取哪个正则表达式组。如果 `index` 为0，则返回整个匹配的字符串。 + +数据类型: `String`. + +**示例** + +``` sql +SELECT + regexpExtract('100-200', '(\\d+)-(\\d+)', 1), + regexpExtract('100-200', '(\\d+)-(\\d+)', 2), + regexpExtract('100-200', '(\\d+)-(\\d+)', 0), + regexpExtract('100-200', '(\\d+)-(\\d+)'); +``` + +结果： + +``` text +┌─regexpExtract('100-200', '(\\d+)-(\\d+)', 1)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)', 2)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)', 0)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)')─┐ +│ 100 │ 200 │ 100-200 │ 100 │ +└──────────────────────────────────────────────┴──────────────────────────────────────────────┴──────────────────────────────────────────────┴───────────────────────────────────────────┘ +``` + +## hasSubsequence + +如果`needle`是`haystack`的子序列，返回1，否贼返回0。 +子序列是从给定字符串中删除零个或多个元素而不改变剩余元素的顺序得到的序列。 + +**语法** + +``` sql +hasSubsequence(haystack, needle) +``` + +**参数** + +- `haystack` — 被搜索的字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 搜索子序列，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). + +**返回值** + +- 1, 如果`needle`是`haystack`的子序列 +- 0, 如果`needle`不是`haystack`的子序列 + +数据类型: `UInt8`. + +**示例** + +``` sql +SELECT hasSubsequence('garbage', 'arg') ; +``` + +结果： + +``` text +┌─hasSubsequence('garbage', 'arg')─┐ +│ 1 │ +└──────────────────────────────────┘ +``` + +## hasSubsequenceCaseInsensitive +类似于[hasSubsequence](#hasSubsequence)但是不区分大小写。 + +## hasSubsequenceUTF8 + +类似于 [hasSubsequence](#hasSubsequence) 但是假定 `haystack` 和 `needle` 是 UTF-8 编码的字符串。 + +## hasSubsequenceCaseInsensitiveUTF8 + +类似于 [hasSubsequenceUTF8](#hasSubsequenceUTF8) 但是不区分大小写。 From 6f51b3c64dd5eb7c2f493f668ddadc62a28b5b48 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Mar 2024 12:08:58 +0100 Subject: [PATCH 0735/1165] Update 03032_scalars_create_as_select.sql --- tests/queries/0_stateless/03032_scalars_create_as_select.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03032_scalars_create_as_select.sql b/tests/queries/0_stateless/03032_scalars_create_as_select.sql index 5b20b8cf6f9..ae75a30ad8b 100644 --- a/tests/queries/0_stateless/03032_scalars_create_as_select.sql +++ b/tests/queries/0_stateless/03032_scalars_create_as_select.sql @@ -1,2 +1,2 @@ -create or replace table query_run_metric_arrays engine Memory as with (with (select groupUniqArrayArray(['a', 'b']) from numbers(1)) as all_names select all_names) as all_metrics select all_metrics; +create table query_run_metric_arrays engine Memory as with (with (select groupUniqArrayArray(['a', 'b']) from numbers(1)) as all_names select all_names) as all_metrics select all_metrics; select * from query_run_metric_arrays; From 4ad2c78afb21509f7d682618640a2e766dba28e3 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Mar 2024 11:16:21 +0000 Subject: [PATCH 0736/1165] Add analyzer pattern to 00002_log_and_exception_messages_formatting --- .../0_stateless/00002_log_and_exception_messages_formatting.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index dfa11dfcb13..b5710e29be9 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -203,7 +203,7 @@ select with 0.16 as threshold select 'noisy Trace messages', - greatest(coalesce(((select message_format_string, count() from logs where level = 'Trace' and message_format_string not in ('Access granted: {}{}', '{} -> {}') + greatest(coalesce(((select message_format_string, count() from logs where level = 'Trace' and message_format_string not in ('Access granted: {}{}', '{} -> {}', 'Query {} to stage {}{}') group by message_format_string order by count() desc limit 1) as top_message).2, 0) / (select count() from logs), threshold) as r, r <= threshold ? '' : top_message.1; From e32497345013da3ccaf04467d5e52318c82d837f Mon Sep 17 00:00:00 2001 From: Blargian Date: Thu, 28 Mar 2024 12:36:11 +0100 Subject: [PATCH 0737/1165] Standardize function formatting for MultiSearchAllPositionsXYZ and MultiSearchAnyXYZ functions --- .../functions/string-search-functions.md | 221 +++++++++++++++++- 1 file changed, 219 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 22f879c62ae..c4b5fb7c1f5 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -178,11 +178,106 @@ Result: │ [0,13,0] │ └───────────────────────────────────────────────────────────────────┘ ``` +## multiSearchAllPositionsCaseInsensitive + +Like [multiSearchAllPositions](#multisearchallpositions) but ignores case. + +**Syntax** + +```sql +multiSearchAllPositionsCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- Array of the starting position in bytes and counting from 1 (if the substring was found). +- 0 if the substring was not found. + +**Example** + +Query: + +```sql +SELECT multiSearchAllPositionsCaseInsensitive('ClickHouse',['c','h']); +``` + +```response +["1","6"] +``` ## multiSearchAllPositionsUTF8 Like [multiSearchAllPositions](#multiSearchAllPositions) but assumes `haystack` and the `needle`-s are UTF-8 encoded strings. +**Syntax** + +```sql +multiSearchAllPositionsUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- Array of the starting position in bytes and counting from 1 (if the substring was found). +- 0 if the substring was not found. + +**Example** + +Given `ClickHouse` as a UTF-8 string, find the positions of `C` ('\x43') and `H` ('\x48'). + +Query: + +```sql +SELECT multiSearchAllPositionsUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x43','\x48']); +``` + +```response +["1","6"] +``` + +## multiSearchAllPositionsCaseInsensitiveUTF8 + +Like [multiSearchAllPositionsUTF8](#multisearchallpositionsutf8) but ignores case. + +**Syntax** + +```sql +multiSearchAllPositionsCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- Array of the starting position in bytes and counting from 1 (if the substring was found). +- 0 if the substring was not found. + +**Example** + +Given `ClickHouse` as a UTF-8 string, find the positions of `c` (`\x63`) and `h` (`\x68`). + +Query: + +```sql +SELECT multiSearchAllPositionsCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x63','\x68']); +``` + +```response +["1","6"] +``` + ## multiSearchFirstPosition Like `position` but returns the leftmost offset in a `haystack` string which matches any of multiple `needle` strings. @@ -211,12 +306,134 @@ multiSearchFirstIndex(haystack, \[needle1, needle2, …, n Returns 1, if at least one string needlei matches the string `haystack` and 0 otherwise. -Functions `multiSearchAnyCaseInsensitive`, `multiSearchAnyUTF8` and `multiSearchAnyCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. +Functions [`multiSearchAnyCaseInsensitive`](#multiSearchAnyCaseInsensitive), [`multiSearchAnyUTF8`](#multiSearchAnyUTF8) and []`multiSearchAnyCaseInsensitiveUTF8`](#multiSearchAnyCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** ```sql -multiSearchAny(haystack, \[needle1, needle2, …, needlen\]) +multiSearchAny(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- 1, if there was at least one match. +- 0, if there was not at least one match. + +**Example** + +Query: + +```sql +SELECT multiSearchAny('ClickHouse',['C','H']); +``` + +```response +1 +``` + +## multiSearchAnyCaseInsensitive {#multiSearchAnyCaseInsensitive} + +Like [multiSearchAny](#multisearchany) but ignores case. + +**Syntax** + +```sql +multiSearchAnyCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- 1, if there was at least one case-insensitive match. +- 0, if there was not at least one case-insensitive match. + +**Example** + +Query: + +```sql +SELECT multiSearchAnyCaseInsensitive('ClickHouse',['c','h']); +``` + +```response +1 +``` + +## multiSearchAnyUTF8 {#multiSearchAnyUTF8} + +Like [multiSearchAny](#multisearchany) but assumes `haystack` and the `needle`-s are UTF-8 encoded strings. + +*Syntax** + +```sql +multiSearchAnyUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- 1, if there was at least one match. +- 0, if there was not at least one match. + +**Example** + +Given `ClickHouse` as a UTF-8 string, check if there are any `C` ('\x43') or `H` ('\x48') letters in the word. + +Query: + +```sql +SELECT multiSearchAnyUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x43','\x48']); +``` + +```response +1 +``` + +## multiSearchAnyCaseInsensitiveUTF8 {#multiSearchAnyCaseInsensitiveUTF8} + +Like [multiSearchAnyUTF8](#multiSearchAnyUTF8) but ignores case. + +*Syntax** + +```sql +multiSearchAnyCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- 1, if there was at least one case-insensitive match. +- 0, if there was not at least one case-insensitive match. + +**Example** + +Given `ClickHouse` as a UTF-8 string, check if there is any letter `h`(`\x68`) in the word, ignoring case. + +Query: + +```sql +SELECT multiSearchAnyCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x68']); +``` + +```response +1 ``` ## match {#match} From f1ae99b1136218a1ff8cd281293868b7fdabf9fe Mon Sep 17 00:00:00 2001 From: zhongyuankai <872237106@qq.com> Date: Wed, 27 Mar 2024 12:55:04 +0800 Subject: [PATCH 0738/1165] Support ALTER MODIFY SETTING for Memory tables --- .../engines/table-engines/special/memory.md | 5 ++ src/Storages/MemorySettings.cpp | 19 ++++- src/Storages/MemorySettings.h | 3 + src/Storages/StorageMemory.cpp | 50 ++++++++++---- src/Storages/StorageMemory.h | 12 ++-- ...2_storage_memory_modify_settings.reference | 16 +++++ .../03032_storage_memory_modify_settings.sql | 69 +++++++++++++++++++ 7 files changed, 151 insertions(+), 23 deletions(-) create mode 100644 tests/queries/0_stateless/03032_storage_memory_modify_settings.reference create mode 100644 tests/queries/0_stateless/03032_storage_memory_modify_settings.sql diff --git a/docs/en/engines/table-engines/special/memory.md b/docs/en/engines/table-engines/special/memory.md index 19b5c798a76..4f5d8a1d5e0 100644 --- a/docs/en/engines/table-engines/special/memory.md +++ b/docs/en/engines/table-engines/special/memory.md @@ -37,6 +37,11 @@ Upper and lower bounds can be specified to limit Memory engine table size, effec - `max_rows_to_keep` — Maximum rows to keep within memory table where oldest rows are deleted on each insertion (i.e circular buffer). Max rows can exceed the stated limit if the oldest batch of rows to remove falls under the `min_rows_to_keep` limit when adding a large block. - Default value: `0` +**Modify settings** +```sql +ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; +``` + ## Usage {#usage} diff --git a/src/Storages/MemorySettings.cpp b/src/Storages/MemorySettings.cpp index f5e182b3484..4968f2b3b16 100644 --- a/src/Storages/MemorySettings.cpp +++ b/src/Storages/MemorySettings.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -11,6 +10,7 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_SETTING; + extern const int SETTING_CONSTRAINT_VIOLATION; } IMPLEMENT_SETTINGS_TRAITS(memorySettingsTraits, MEMORY_SETTINGS) @@ -32,5 +32,22 @@ void MemorySettings::loadFromQuery(ASTStorage & storage_def) } } +ASTPtr MemorySettings::getSettingsChangesQuery() +{ + auto settings_ast = std::make_shared(); + settings_ast->is_standalone = false; + for (const auto & change : changes()) + settings_ast->changes.push_back(change); + + return settings_ast; +} + +void MemorySettings::sanityCheck() const +{ + if (min_bytes_to_keep > max_bytes_to_keep + || min_rows_to_keep > max_rows_to_keep) + throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Min. bytes / rows must be set with a max."); +} + } diff --git a/src/Storages/MemorySettings.h b/src/Storages/MemorySettings.h index ac6cdf73329..f650746c4b2 100644 --- a/src/Storages/MemorySettings.h +++ b/src/Storages/MemorySettings.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -24,6 +25,8 @@ DECLARE_SETTINGS_TRAITS(memorySettingsTraits, MEMORY_SETTINGS) struct MemorySettings : public BaseSettings { void loadFromQuery(ASTStorage & storage_def); + ASTPtr getSettingsChangesQuery(); + void sanityCheck() const; }; } diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index c6222d2124e..2de2878002a 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -76,7 +76,7 @@ public: convertDynamicColumnsToTuples(block, storage_snapshot); } - if (storage.compress) + if (storage.getMemorySettingsRef().compress) { Block compressed_block; for (const auto & elem : block) @@ -106,15 +106,16 @@ public: auto new_data = std::make_unique(*(storage.data.get())); UInt64 new_total_rows = storage.total_size_rows.load(std::memory_order_relaxed) + inserted_rows; UInt64 new_total_bytes = storage.total_size_bytes.load(std::memory_order_relaxed) + inserted_bytes; + auto & memory_settings = storage.getMemorySettingsRef(); while (!new_data->empty() - && ((storage.max_bytes_to_keep && new_total_bytes > storage.max_bytes_to_keep) - || (storage.max_rows_to_keep && new_total_rows > storage.max_rows_to_keep))) + && ((memory_settings.max_bytes_to_keep && new_total_bytes > memory_settings.max_bytes_to_keep) + || (memory_settings.max_rows_to_keep && new_total_rows > memory_settings.max_rows_to_keep))) { Block oldest_block = new_data->front(); UInt64 rows_to_remove = oldest_block.rows(); UInt64 bytes_to_remove = oldest_block.allocatedBytes(); - if (new_total_bytes - bytes_to_remove < storage.min_bytes_to_keep - || new_total_rows - rows_to_remove < storage.min_rows_to_keep) + if (new_total_bytes - bytes_to_remove < memory_settings.min_bytes_to_keep + || new_total_rows - rows_to_remove < memory_settings.min_rows_to_keep) { break; // stop - removing next block will put us under min_bytes / min_rows threshold } @@ -145,15 +146,16 @@ StorageMemory::StorageMemory( ColumnsDescription columns_description_, ConstraintsDescription constraints_, const String & comment, - const MemorySettings & settings) - : IStorage(table_id_), data(std::make_unique()), compress(settings.compress), - min_rows_to_keep(settings.min_rows_to_keep), max_rows_to_keep(settings.max_rows_to_keep), - min_bytes_to_keep(settings.min_bytes_to_keep), max_bytes_to_keep(settings.max_bytes_to_keep) + const MemorySettings & memory_settings_) + : IStorage(table_id_) + , data(std::make_unique()) + , memory_settings(memory_settings_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(std::move(columns_description_)); storage_metadata.setConstraints(std::move(constraints_)); storage_metadata.setComment(comment); + storage_metadata.setSettingsChanges(memory_settings.getSettingsChangesQuery()); setInMemoryMetadata(storage_metadata); } @@ -239,7 +241,7 @@ void StorageMemory::mutate(const MutationCommands & commands, ContextPtr context Block block; while (executor.pull(block)) { - if (compress) + if (memory_settings.compress) for (auto & elem : block) elem.column = elem.column->compress(); @@ -294,6 +296,25 @@ void StorageMemory::truncate( total_size_rows.store(0, std::memory_order_relaxed); } +void StorageMemory::alter(const DB::AlterCommands & params, DB::ContextPtr context, DB::IStorage::AlterLockHolder & /*alter_lock_holder*/) +{ + auto table_id = getStorageID(); + StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); + params.apply(new_metadata, context); + + if (params.isSettingsAlter()) + { + auto & settings_changes = new_metadata.settings_changes->as(); + auto copy = memory_settings; + copy.applyChanges(settings_changes.changes); + copy.sanityCheck(); + memory_settings = std::move(copy); + } + + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); + setInMemoryMetadata(new_metadata); +} + namespace { @@ -499,7 +520,7 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat while (auto block = block_in.read()) { - if (compress) + if (memory_settings.compress) { Block compressed_block; for (const auto & elem : block) @@ -534,7 +555,8 @@ void StorageMemory::checkAlterIsPossible(const AlterCommands & commands, Context { if (command.type != AlterCommand::Type::ADD_COLUMN && command.type != AlterCommand::Type::MODIFY_COLUMN && command.type != AlterCommand::Type::DROP_COLUMN && command.type != AlterCommand::Type::COMMENT_COLUMN - && command.type != AlterCommand::Type::COMMENT_TABLE && command.type != AlterCommand::Type::RENAME_COLUMN) + && command.type != AlterCommand::Type::COMMENT_TABLE && command.type != AlterCommand::Type::RENAME_COLUMN + && command.type != AlterCommand::Type::MODIFY_SETTING) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Alter of type '{}' is not supported by storage {}", command.type, getName()); } @@ -566,9 +588,7 @@ void registerStorageMemory(StorageFactory & factory) if (has_settings) settings.loadFromQuery(*args.storage_def); - if (settings.min_bytes_to_keep > settings.max_bytes_to_keep - || settings.min_rows_to_keep > settings.max_rows_to_keep) - throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Min. bytes / rows must be set with a max."); + settings.sanityCheck(); return std::make_shared(args.table_id, args.columns, args.constraints, args.comment, settings); }, diff --git a/src/Storages/StorageMemory.h b/src/Storages/StorageMemory.h index 13f1c971d82..50581aa0d61 100644 --- a/src/Storages/StorageMemory.h +++ b/src/Storages/StorageMemory.h @@ -31,7 +31,7 @@ public: ColumnsDescription columns_description_, ConstraintsDescription constraints_, const String & comment, - const MemorySettings & settings = MemorySettings()); + const MemorySettings & memory_settings_ = MemorySettings()); String getName() const override { return "Memory"; } @@ -46,6 +46,8 @@ public: StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const override; + const MemorySettings & getMemorySettingsRef() const { return memory_settings; } + void read( QueryPlan & query_plan, const Names & column_names, @@ -78,6 +80,7 @@ public: void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) override; void checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const override; + void alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & alter_lock_holder) override; std::optional totalRows(const Settings &) const override; std::optional totalBytes(const Settings &) const override; @@ -134,12 +137,7 @@ private: std::atomic total_size_bytes = 0; std::atomic total_size_rows = 0; - bool compress; - UInt64 min_rows_to_keep; - UInt64 max_rows_to_keep; - UInt64 min_bytes_to_keep; - UInt64 max_bytes_to_keep; - + MemorySettings memory_settings; friend class ReadFromMemoryStorageStep; }; diff --git a/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference b/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference new file mode 100644 index 00000000000..20dda4fa15a --- /dev/null +++ b/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference @@ -0,0 +1,16 @@ +TESTING BYTES +8192 +9216 +9216 +65536 +TESTING ROWS +50 +1000 +1020 +1100 +TESTING NO CIRCULAR-BUFFER +8192 +9216 +17408 +82944 +TESTING INVALID SETTINGS diff --git a/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql new file mode 100644 index 00000000000..34be327175e --- /dev/null +++ b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql @@ -0,0 +1,69 @@ +SET max_block_size = 65409; -- Default value + +DROP TABLE IF EXISTS memory; +CREATE TABLE memory (i UInt32) ENGINE = Memory; + +ALTER TABLE memory MODIFY SETTING min_bytes_to_keep = 4096, max_bytes_to_keep = 16384; + +SELECT 'TESTING BYTES'; +/* 1. testing oldest block doesn't get deleted because of min-threshold */ +INSERT INTO memory SELECT * FROM numbers(0, 1600); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +/* 2. adding block that doesn't get deleted */ +INSERT INTO memory SELECT * FROM numbers(1000, 100); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +/* 3. testing oldest block gets deleted - 9216 bytes - 1100 */ +INSERT INTO memory SELECT * FROM numbers(9000, 1000); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +/* 4.check large block over-writes all bytes / rows */ +INSERT INTO memory SELECT * FROM numbers(9000, 10000); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +DROP TABLE IF EXISTS memory; +CREATE TABLE memory (i UInt32) ENGINE = Memory; + +ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; + +SELECT 'TESTING ROWS'; +/* 1. add normal number of rows */ +INSERT INTO memory SELECT * FROM numbers(0, 50); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +/* 2. table should have 1000 */ +INSERT INTO memory SELECT * FROM numbers(50, 950); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +/* 3. table should have 1020 - removed first 50 */ +INSERT INTO memory SELECT * FROM numbers(2000, 70); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +/* 4. check large block over-writes all rows */ +INSERT INTO memory SELECT * FROM numbers(3000, 1100); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +SELECT 'TESTING NO CIRCULAR-BUFFER'; +DROP TABLE IF EXISTS memory; +CREATE TABLE memory (i UInt32) ENGINE = Memory; + +INSERT INTO memory SELECT * FROM numbers(0, 1600); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +INSERT INTO memory SELECT * FROM numbers(1000, 100); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +INSERT INTO memory SELECT * FROM numbers(9000, 1000); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +INSERT INTO memory SELECT * FROM numbers(9000, 10000); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +SELECT 'TESTING INVALID SETTINGS'; +DROP TABLE IF EXISTS memory; +CREATE TABLE memory (i UInt32) ENGINE = Memory; +ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100; -- { serverError 452 } +ALTER TABLE memory MODIFY SETTING min_bytes_to_keep = 100; -- { serverError 452 } + +DROP TABLE memory; \ No newline at end of file From a376a7354fc863151aecf774df5cd82a73bd3b94 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 28 Mar 2024 15:04:14 +0300 Subject: [PATCH 0739/1165] Fixed code review issues --- .../AggregateFunctionUniq.h | 10 ++++---- .../Combinators/AggregateFunctionArray.h | 4 ++-- .../Combinators/AggregateFunctionIf.h | 4 ++-- .../Combinators/AggregateFunctionMerge.h | 4 ++-- .../Combinators/AggregateFunctionNull.h | 4 ++-- .../Combinators/AggregateFunctionState.h | 4 ++-- src/AggregateFunctions/IAggregateFunction.h | 4 ++-- src/AggregateFunctions/UniqExactSet.h | 11 ++++----- src/Interpreters/Aggregator.cpp | 24 +++++++++---------- src/Interpreters/Aggregator.h | 18 +++++++------- .../AggregatingInOrderTransform.cpp | 2 +- .../Transforms/AggregatingTransform.cpp | 10 ++++---- ...gingAggregatedMemoryEfficientTransform.cpp | 2 +- .../Transforms/MergingAggregatedTransform.cpp | 2 +- src/Processors/Transforms/RollupTransform.cpp | 3 ++- 15 files changed, 52 insertions(+), 54 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionUniq.h b/src/AggregateFunctions/AggregateFunctionUniq.h index c53b5e3bdb7..cef23f766c7 100644 --- a/src/AggregateFunctions/AggregateFunctionUniq.h +++ b/src/AggregateFunctions/AggregateFunctionUniq.h @@ -459,7 +459,7 @@ public: bool isParallelizeMergePrepareNeeded() const override { return is_parallelize_merge_prepare_needed; } - void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { if constexpr (is_parallelize_merge_prepare_needed) { @@ -485,10 +485,10 @@ public: bool isAbleToParallelizeMerge() const override { return is_able_to_parallelize_merge; } bool canOptimizeEqualKeysRanges() const override { return !is_able_to_parallelize_merge; } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena *) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena *) const override { if constexpr (is_able_to_parallelize_merge) - this->data(place).set.merge(this->data(rhs).set, &thread_pool, is_cancelled); + this->data(place).set.merge(this->data(rhs).set, &thread_pool, &is_cancelled); else this->data(place).set.merge(this->data(rhs).set); } @@ -579,10 +579,10 @@ public: bool isAbleToParallelizeMerge() const override { return is_able_to_parallelize_merge; } bool canOptimizeEqualKeysRanges() const override { return !is_able_to_parallelize_merge; } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena *) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena *) const override { if constexpr (is_able_to_parallelize_merge) - this->data(place).set.merge(this->data(rhs).set, &thread_pool, is_cancelled); + this->data(place).set.merge(this->data(rhs).set, &thread_pool, &is_cancelled); else this->data(place).set.merge(this->data(rhs).set); } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h index 9dc5e274dab..1940985f8e3 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h @@ -144,12 +144,12 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override { nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h index 91dcfa4db0b..a893fc91780 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h @@ -167,12 +167,12 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override { nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h index 5bc478116e0..4a39ec0ab87 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h @@ -113,12 +113,12 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override { nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h index eef5f8bf66b..306e293cae7 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h @@ -154,7 +154,7 @@ public: bool isAbleToParallelizeMerge() const override { return nested_function->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_function->canOptimizeEqualKeysRanges(); } - void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { AggregateDataPtrs nested_places(places.begin(), places.end()); for (auto & nested_place : nested_places) @@ -163,7 +163,7 @@ public: nested_function->parallelizeMergePrepare(nested_places, thread_pool, is_cancelled); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override { nested_function->merge(nestedPlace(place), nestedPlace(rhs), thread_pool, is_cancelled, arena); } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionState.h b/src/AggregateFunctions/Combinators/AggregateFunctionState.h index 7b2933d42c9..5f2eb647c92 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionState.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionState.h @@ -94,12 +94,12 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic * is_cancelled) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic * is_cancelled, Arena * arena) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override { nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index b33d4b20a16..97e0e89aee9 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -151,7 +151,7 @@ public: virtual bool isParallelizeMergePrepareNeeded() const { return false; } - virtual void parallelizeMergePrepare(AggregateDataPtrs & /*places*/, ThreadPool & /*thread_pool*/, std::atomic * /*is_cancelled*/) const + virtual void parallelizeMergePrepare(AggregateDataPtrs & /*places*/, ThreadPool & /*thread_pool*/, std::atomic & /*is_cancelled*/) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "parallelizeMergePrepare() with thread pool parameter isn't implemented for {} ", getName()); } @@ -168,7 +168,7 @@ public: /// Should be used only if isAbleToParallelizeMerge() returned true. virtual void - merge(AggregateDataPtr __restrict /*place*/, ConstAggregateDataPtr /*rhs*/, ThreadPool & /*thread_pool*/, std::atomic * /*is_cancelled*/, Arena * /*arena*/) const + merge(AggregateDataPtr __restrict /*place*/, ConstAggregateDataPtr /*rhs*/, ThreadPool & /*thread_pool*/, std::atomic & /*is_cancelled*/, Arena * /*arena*/) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "merge() with thread pool parameter isn't implemented for {} ", getName()); } diff --git a/src/AggregateFunctions/UniqExactSet.h b/src/AggregateFunctions/UniqExactSet.h index 18b0b830cb8..2ae8c3a8386 100644 --- a/src/AggregateFunctions/UniqExactSet.h +++ b/src/AggregateFunctions/UniqExactSet.h @@ -37,7 +37,7 @@ public: /// In merge, if one of the lhs and rhs is twolevelset and the other is singlelevelset, then the singlelevelset will need to convertToTwoLevel(). /// It's not in parallel and will cost extra large time if the thread_num is large. /// This method will convert all the SingleLevelSet to TwoLevelSet in parallel if the hashsets are not all singlelevel or not all twolevel. - static void parallelizeMergePrepare(const std::vector & data_vec, ThreadPool & thread_pool, std::atomic * is_cancelled) + static void parallelizeMergePrepare(const std::vector & data_vec, ThreadPool & thread_pool, std::atomic & is_cancelled) { UInt64 single_level_set_num = 0; UInt64 all_single_hash_size = 0; @@ -63,7 +63,7 @@ public: try { auto data_vec_atomic_index = std::make_shared(0); - auto thread_func = [data_vec, data_vec_atomic_index, is_cancelled, thread_group = CurrentThread::getGroup()]() + auto thread_func = [data_vec, data_vec_atomic_index, &is_cancelled, thread_group = CurrentThread::getGroup()]() { SCOPE_EXIT_SAFE( if (thread_group) @@ -76,7 +76,7 @@ public: while (true) { - if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) + if (is_cancelled.load(std::memory_order_seq_cst)) return; const auto i = data_vec_atomic_index->fetch_add(1); @@ -117,9 +117,6 @@ public: { for (size_t i = 0; i < rhs.NUM_BUCKETS; ++i) { - if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) - return; - lhs.impls[i].merge(rhs.impls[i]); } } @@ -141,7 +138,7 @@ public: while (true) { - if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) + if (is_cancelled->load(std::memory_order_seq_cst)) return; const auto bucket = next_bucket_to_merge->fetch_add(1); diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 837b4e47fba..ab8cec864ae 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1436,7 +1436,7 @@ void NO_INLINE Aggregator::mergeOnIntervalWithoutKey( size_t row_begin, size_t row_end, const AggregateColumnsConstData & aggregate_columns_data, - std::atomic * is_cancelled) const + std::atomic & is_cancelled) const { /// `data_variants` will destroy the states of aggregate functions in the destructor data_variants.aggregator = this; @@ -1752,7 +1752,7 @@ Block Aggregator::mergeAndConvertOneBucketToBlock( Arena * arena, bool final, Int32 bucket, - std::atomic * is_cancelled) const + std::atomic & is_cancelled) const { auto & merged_data = *variants[0]; auto method = merged_data.type; @@ -1762,8 +1762,8 @@ Block Aggregator::mergeAndConvertOneBucketToBlock( #define M(NAME) \ else if (method == AggregatedDataVariants::Type::NAME) \ { \ - mergeBucketImpl(variants, bucket, arena); \ - if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) \ + mergeBucketImpl(variants, bucket, arena, is_cancelled); \ + if (is_cancelled.load(std::memory_order_seq_cst)) \ return {}; \ block = convertOneBucketToBlock(merged_data, *merged_data.NAME, arena, final, bucket); \ } @@ -2638,7 +2638,7 @@ void NO_INLINE Aggregator::mergeDataOnlyExistingKeysImpl( void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( ManyAggregatedDataVariants & non_empty_data, - std::atomic * is_cancelled) const + std::atomic & is_cancelled) const { ThreadPool thread_pool{CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, params.max_threads}; @@ -2746,7 +2746,7 @@ void NO_INLINE Aggregator::mergeSingleLevelDataImpl( template void NO_INLINE Aggregator::mergeBucketImpl( - ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic * is_cancelled) const + ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic & is_cancelled) const { /// We merge all aggregation results to the first. AggregatedDataVariantsPtr & res = data[0]; @@ -2756,7 +2756,7 @@ void NO_INLINE Aggregator::mergeBucketImpl( for (size_t result_num = 1, size = data.size(); result_num < size; ++result_num) { - if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) + if (is_cancelled.load(std::memory_order_seq_cst)) return; AggregatedDataVariants & current = *data[result_num]; @@ -2958,7 +2958,7 @@ void NO_INLINE Aggregator::mergeStreamsImpl( void NO_INLINE Aggregator::mergeBlockWithoutKeyStreamsImpl( Block block, AggregatedDataVariants & result, - std::atomic * is_cancelled) const + std::atomic & is_cancelled) const { AggregateColumnsConstData aggregate_columns = params.makeAggregateColumnsData(block); mergeWithoutKeyStreamsImpl(result, 0, block.rows(), aggregate_columns, is_cancelled); @@ -2969,7 +2969,7 @@ void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( size_t row_begin, size_t row_end, const AggregateColumnsConstData & aggregate_columns_data, - std::atomic * is_cancelled) const + std::atomic & is_cancelled) const { using namespace CurrentMetrics; @@ -3005,7 +3005,7 @@ void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( } -bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys, std::atomic * is_cancelled) const +bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys, std::atomic & is_cancelled) const { /// `result` will destroy the states of aggregate functions in the destructor result.aggregator = this; @@ -3075,7 +3075,7 @@ bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool } -void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads, std::atomic * is_cancelled) +void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads, std::atomic & is_cancelled) { if (bucket_to_blocks.empty()) return; @@ -3207,7 +3207,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari } -Block Aggregator::mergeBlocks(BlocksList & blocks, bool final, std::atomic * is_cancelled) +Block Aggregator::mergeBlocks(BlocksList & blocks, bool final, std::atomic & is_cancelled) { if (blocks.empty()) return {}; diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 4bce700a099..406d28597cf 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -269,7 +269,7 @@ public: bool mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys, - std::atomic * is_cancelled) const; + std::atomic & is_cancelled) const; void mergeOnBlockSmall( AggregatedDataVariants & result, @@ -283,7 +283,7 @@ public: size_t row_begin, size_t row_end, const AggregateColumnsConstData & aggregate_columns_data, - std::atomic * is_cancelled) const; + std::atomic & is_cancelled) const; /** Convert the aggregation data structure into a block. * If overflow_row = true, then aggregates for rows that are not included in max_rows_to_group_by are put in the first block. @@ -298,13 +298,13 @@ public: using BucketToBlocks = std::map; /// Merge partially aggregated blocks separated to buckets into one data structure. - void mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads, std::atomic * is_cancelled); + void mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads, std::atomic & is_cancelled); /// Merge several partially aggregated blocks into one. /// Precondition: for all blocks block.info.is_overflows flag must be the same. /// (either all blocks are from overflow data or none blocks are). /// The resulting block has the same value of is_overflows flag. - Block mergeBlocks(BlocksList & blocks, bool final, std::atomic * is_cancelled); + Block mergeBlocks(BlocksList & blocks, bool final, std::atomic & is_cancelled); /** Split block with partially-aggregated data to many blocks, as if two-level method of aggregation was used. * This is needed to simplify merging of that data with other results, that are already two-level. @@ -491,7 +491,7 @@ private: void mergeWithoutKeyDataImpl( ManyAggregatedDataVariants & non_empty_data, - std::atomic * is_cancelled) const; + std::atomic & is_cancelled) const; template void mergeSingleLevelDataImpl( @@ -546,7 +546,7 @@ private: Arena * arena, bool final, Int32 bucket, - std::atomic * is_cancelled = nullptr) const; + std::atomic & is_cancelled) const; Block prepareBlockAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final, bool is_overflows) const; BlocksList prepareBlocksAndFillTwoLevel(AggregatedDataVariants & data_variants, bool final, ThreadPool * thread_pool) const; @@ -603,18 +603,18 @@ private: void mergeBlockWithoutKeyStreamsImpl( Block block, AggregatedDataVariants & result, - std::atomic * is_cancelled) const; + std::atomic & is_cancelled) const; void mergeWithoutKeyStreamsImpl( AggregatedDataVariants & result, size_t row_begin, size_t row_end, const AggregateColumnsConstData & aggregate_columns_data, - std::atomic * is_cancelled) const; + std::atomic & is_cancelled) const; template void mergeBucketImpl( - ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic * is_cancelled = nullptr) const; + ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic & is_cancelled) const; template void convertBlockToTwoLevelImpl( diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index f959b2b01b4..9ffe15d0f85 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -160,7 +160,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) if (group_by_key) params->aggregator.mergeOnBlockSmall(variants, key_begin, key_end, aggregate_columns_data, key_columns_raw); else - params->aggregator.mergeOnIntervalWithoutKey(variants, key_begin, key_end, aggregate_columns_data, &is_cancelled); + params->aggregator.mergeOnIntervalWithoutKey(variants, key_begin, key_end, aggregate_columns_data, is_cancelled); } else { diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 767448edc64..b48d435720a 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -132,7 +132,7 @@ protected: return {}; } - Block block = params->aggregator.mergeAndConvertOneBucketToBlock(*data, arena, params->final, bucket_num, &shared_data->is_cancelled); + Block block = params->aggregator.mergeAndConvertOneBucketToBlock(*data, arena, params->final, bucket_num, shared_data->is_cancelled); Chunk chunk = convertToChunk(block); shared_data->is_bucket_processed[bucket_num] = true; @@ -350,7 +350,7 @@ public: for (auto & input : inputs) input.close(); - shared_data->is_cancelled.store(true); + shared_data->is_cancelled.store(true, std::memory_order_seq_cst); return Status::Finished; } @@ -472,7 +472,7 @@ private: if (first->type == AggregatedDataVariants::Type::without_key || params->params.overflow_row) { - params->aggregator.mergeWithoutKeyDataImpl(*data, &shared_data->is_cancelled); + params->aggregator.mergeWithoutKeyDataImpl(*data, shared_data->is_cancelled); auto block = params->aggregator.prepareBlockAndFillWithoutKey( *first, params->final, first->type != AggregatedDataVariants::Type::without_key); @@ -692,7 +692,7 @@ void AggregatingTransform::consume(Chunk chunk) { auto block = getInputs().front().getHeader().cloneWithColumns(chunk.detachColumns()); block = materializeBlock(block); - if (!params->aggregator.mergeOnBlock(block, variants, no_more_keys, &is_cancelled)) + if (!params->aggregator.mergeOnBlock(block, variants, no_more_keys, is_cancelled)) is_consume_finished = true; } else @@ -712,7 +712,7 @@ void AggregatingTransform::initGenerate() if (variants.empty() && params->params.keys_size == 0 && !params->params.empty_result_for_aggregation_by_empty_set) { if (params->params.only_merge) - params->aggregator.mergeOnBlock(getInputs().front().getHeader(), variants, no_more_keys, &is_cancelled); + params->aggregator.mergeOnBlock(getInputs().front().getHeader(), variants, no_more_keys, is_cancelled); else params->aggregator.executeOnBlock(getInputs().front().getHeader(), variants, key_columns, aggregate_columns, no_more_keys); } diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp index 3bfd7874ac7..fc40c6894bb 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -363,7 +363,7 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk) res_info->chunk_num = chunks_to_merge->chunk_num; chunk.setChunkInfo(std::move(res_info)); - auto block = params->aggregator.mergeBlocks(blocks_list, params->final, &is_cancelled); + auto block = params->aggregator.mergeBlocks(blocks_list, params->final, is_cancelled); if (!required_sort_description.empty()) sortBlock(block, required_sort_description); diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 64207093568..ad723da7527 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -72,7 +72,7 @@ Chunk MergingAggregatedTransform::generate() next_block = blocks.begin(); /// TODO: this operation can be made async. Add async for IAccumulatingTransform. - params->aggregator.mergeBlocks(std::move(bucket_to_blocks), data_variants, max_threads, &is_cancelled); + params->aggregator.mergeBlocks(std::move(bucket_to_blocks), data_variants, max_threads, is_cancelled); blocks = params->aggregator.convertToBlocks(data_variants, params->final, max_threads); next_block = blocks.begin(); } diff --git a/src/Processors/Transforms/RollupTransform.cpp b/src/Processors/Transforms/RollupTransform.cpp index 20ee91a203a..bc5a7aeb7bf 100644 --- a/src/Processors/Transforms/RollupTransform.cpp +++ b/src/Processors/Transforms/RollupTransform.cpp @@ -57,7 +57,8 @@ Chunk GroupByModifierTransform::merge(Chunks && chunks, bool is_input, bool fina for (auto & chunk : chunks) blocks.emplace_back(header.cloneWithColumns(chunk.detachColumns())); - auto current_block = is_input ? params->aggregator.mergeBlocks(blocks, final, &is_cancelled) : output_aggregator->mergeBlocks(blocks, final, &is_cancelled); + auto & aggregator = is_input ? params->aggregator : *output_aggregator; + auto current_block = aggregator.mergeBlocks(blocks, final, is_cancelled); auto num_rows = current_block.rows(); return Chunk(current_block.getColumns(), num_rows); } From f4c5be3bb4bc999c65eb040ced98be88982d12f0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Mar 2024 14:04:31 +0100 Subject: [PATCH 0740/1165] Fix CREATE TABLE w/o columns definition for ReplicatedMergeTree Signed-off-by: Azat Khuzhin --- src/Storages/MergeTree/registerStorageMergeTree.cpp | 4 ++-- .../03032_rmt_create_columns_from_replica.reference | 7 +++++++ .../0_stateless/03032_rmt_create_columns_from_replica.sql | 5 +++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03032_rmt_create_columns_from_replica.reference create mode 100644 tests/queries/0_stateless/03032_rmt_create_columns_from_replica.sql diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index e89547952d0..9a3c17923d8 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -588,7 +588,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) FunctionNameNormalizer().visit(partition_key.get()); auto primary_key_asts = metadata.primary_key.expression_list_ast->children; metadata.minmax_count_projection.emplace(ProjectionDescription::getMinMaxCountProjection( - args.columns, partition_key, minmax_columns, primary_key_asts, context)); + columns, partition_key, minmax_columns, primary_key_asts, context)); if (args.storage_def->sample_by) metadata.sampling_key = KeyDescription::getKeyFromAST(args.storage_def->sample_by->ptr(), metadata.columns, context); @@ -697,7 +697,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) FunctionNameNormalizer().visit(partition_key.get()); auto primary_key_asts = metadata.primary_key.expression_list_ast->children; metadata.minmax_count_projection.emplace(ProjectionDescription::getMinMaxCountProjection( - args.columns, partition_key, minmax_columns, primary_key_asts, context)); + columns, partition_key, minmax_columns, primary_key_asts, context)); const auto * ast = engine_args[arg_num]->as(); if (ast && ast->value.getType() == Field::Types::UInt64) diff --git a/tests/queries/0_stateless/03032_rmt_create_columns_from_replica.reference b/tests/queries/0_stateless/03032_rmt_create_columns_from_replica.reference new file mode 100644 index 00000000000..6dda968659d --- /dev/null +++ b/tests/queries/0_stateless/03032_rmt_create_columns_from_replica.reference @@ -0,0 +1,7 @@ +CREATE TABLE default.data_r2 +( + `key` Int32 +) +ENGINE = ReplicatedMergeTree('/tables/default', 'r2') +ORDER BY tuple() +SETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/03032_rmt_create_columns_from_replica.sql b/tests/queries/0_stateless/03032_rmt_create_columns_from_replica.sql new file mode 100644 index 00000000000..e08db0bdacf --- /dev/null +++ b/tests/queries/0_stateless/03032_rmt_create_columns_from_replica.sql @@ -0,0 +1,5 @@ +drop table if exists data_r1; +drop table if exists data_r2; +create table data_r1 (key Int) engine=ReplicatedMergeTree('/tables/{database}', 'r1') order by tuple(); +create table data_r2 engine=ReplicatedMergeTree('/tables/{database}', 'r2') order by tuple(); +show create data_r2 format LineAsString; From 7d20467f59b629a1d690a3347b7cd551c742496b Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:35:43 +0100 Subject: [PATCH 0741/1165] Fix test reference --- .../0_stateless/03013_ignore_drop_queries_probability.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/03013_ignore_drop_queries_probability.reference b/tests/queries/0_stateless/03013_ignore_drop_queries_probability.reference index d81cc0710eb..daaac9e3030 100644 --- a/tests/queries/0_stateless/03013_ignore_drop_queries_probability.reference +++ b/tests/queries/0_stateless/03013_ignore_drop_queries_probability.reference @@ -1 +1,2 @@ 42 +42 From 1a75d3ed46d0a2fe6a596afbc27ae20ff97427fb Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Mar 2024 13:58:58 +0000 Subject: [PATCH 0742/1165] Updating test. --- .../02802_with_cube_with_totals.reference | 30 +++++++++++++++++++ .../02802_with_cube_with_totals.sql | 1 + 2 files changed, 31 insertions(+) diff --git a/tests/queries/0_stateless/02802_with_cube_with_totals.reference b/tests/queries/0_stateless/02802_with_cube_with_totals.reference index c7b7b570456..206c32e562b 100644 --- a/tests/queries/0_stateless/02802_with_cube_with_totals.reference +++ b/tests/queries/0_stateless/02802_with_cube_with_totals.reference @@ -1,5 +1,35 @@ ((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 ((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 ((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 \N diff --git a/tests/queries/0_stateless/02802_with_cube_with_totals.sql b/tests/queries/0_stateless/02802_with_cube_with_totals.sql index 77adb68eb4b..168e4d61b68 100644 --- a/tests/queries/0_stateless/02802_with_cube_with_totals.sql +++ b/tests/queries/0_stateless/02802_with_cube_with_totals.sql @@ -1,2 +1,3 @@ +set allow_experimental_analyzer=1; SELECT tuple((2147483648, (-0., 1.1754943508222875e-38, 2147483646, '-9223372036854775808', NULL))), toInt128(0.0001) GROUP BY ((256, toInt64(1.1754943508222875e-38), NULL), NULL, -0., ((65535, '-92233720368547758.07'), 0.9999), tuple(((1., 3.4028234663852886e38, '1', 0.5), NULL, tuple('0.1')))) WITH CUBE WITH TOTALS; SELECT NULL GROUP BY toUUID(NULL, '0', NULL, '0.0000065535'), 1 WITH CUBE WITH TOTALS; From 863f479ff025532ea8a601c7081b65931197b98e Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Thu, 28 Mar 2024 14:00:04 +0000 Subject: [PATCH 0743/1165] CI: fix for timeout env variable --- tests/ci/ci.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 17552377971..0f57d3ba991 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1761,9 +1761,10 @@ def _run_test(job_name: str, run_command: str) -> int: run_command or CI_CONFIG.get_job_config(job_name).run_command ), "Run command must be provided as input argument or be configured in job config" + if CI_CONFIG.get_job_config(job_name).timeout: + os.environ["KILL_TIMEOUT"] = str(CI_CONFIG.get_job_config(job_name).timeout) + if not run_command: - if CI_CONFIG.get_job_config(job_name).timeout: - os.environ["KILL_TIMEOUT"] = str(CI_CONFIG.get_job_config(job_name).timeout) run_command = "/".join( (os.path.dirname(__file__), CI_CONFIG.get_job_config(job_name).run_command) ) From 4b990e732cd0e0cb30e03b4b930ce3a85ef448c4 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 28 Mar 2024 15:22:26 +0100 Subject: [PATCH 0744/1165] Fix style --- src/Storages/StorageSet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 8561c3f3aa8..54218351cf1 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -248,7 +248,7 @@ void StorageSetOrJoinBase::restore() static const auto file_suffix_size = strlen(".bin"); using FilePriority = std::pair; - std::priority_queue, std::greater> backup_files; + std::priority_queue, std::greater<>> backup_files; for (auto dir_it{disk->iterateDirectory(path)}; dir_it->isValid(); dir_it->next()) { const auto & name = dir_it->name(); From e2e0584d8d3079660ae3976c7e91b17d69d1ec18 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 28 Mar 2024 15:42:12 +0100 Subject: [PATCH 0745/1165] Add test for #53352 (#61886) --- ...031_filter_float64_logical_error.reference | 0 .../03031_filter_float64_logical_error.sql | 40 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tests/queries/0_stateless/03031_filter_float64_logical_error.reference create mode 100644 tests/queries/0_stateless/03031_filter_float64_logical_error.sql diff --git a/tests/queries/0_stateless/03031_filter_float64_logical_error.reference b/tests/queries/0_stateless/03031_filter_float64_logical_error.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03031_filter_float64_logical_error.sql b/tests/queries/0_stateless/03031_filter_float64_logical_error.sql new file mode 100644 index 00000000000..59d4a06c8f7 --- /dev/null +++ b/tests/queries/0_stateless/03031_filter_float64_logical_error.sql @@ -0,0 +1,40 @@ +CREATE TABLE 03031_test +( + `id` UInt64, + `value_1` String, + `value_2` String, + `value_3` String, + INDEX value_1_idx value_1 TYPE bloom_filter GRANULARITY 1, + INDEX value_2_idx value_2 TYPE ngrambf_v1(3, 512, 2, 0) GRANULARITY 1, + INDEX value_3_idx value_3 TYPE tokenbf_v1(512, 3, 0) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO 03031_test SELECT + number, + toString(number), + toString(number), + toString(number) +FROM numbers(10); + +SELECT + count('9223372036854775806'), + 7 +FROM 03031_test +PREWHERE (id = NULL) AND 1024 +WHERE 0.0001 +GROUP BY '0.03' + WITH ROLLUP +SETTINGS force_primary_key = 1, force_data_skipping_indices = 'value_1_idx, value_2_idx', allow_experimental_analyzer=0; + + +SELECT + count('9223372036854775806'), + 7 +FROM 03031_test +PREWHERE (id = NULL) AND 1024 +WHERE 0.0001 +GROUP BY '0.03' + WITH ROLLUP +SETTINGS force_primary_key = 1, force_data_skipping_indices = 'value_1_idx, value_2_idx', allow_experimental_analyzer=1; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } From fb235c0840d380fdf2b7107dc242c18a7f5721a5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Mar 2024 14:44:45 +0000 Subject: [PATCH 0746/1165] Fix optimize_arithmetic_operations_in_aggregate_functions optimization which could change result type. --- .../Passes/RewriteSumFunctionWithSumAndCountPass.cpp | 10 ++++++++-- ..._functions_arithmetic_operations_pass_fix.reference | 1 + ...regate_functions_arithmetic_operations_pass_fix.sql | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp index 3c93bf9e1bf..917256bf4b1 100644 --- a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp +++ b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace DB @@ -83,7 +84,7 @@ public: rhs->getArguments().getNodes().push_back(rhs_count); resolveOrdinaryFunctionNode(*rhs, rhs->getFunctionName()); - const auto new_node = std::make_shared(Poco::toLower(func_plus_minus_node->getFunctionName())); + auto new_node = std::make_shared(Poco::toLower(func_plus_minus_node->getFunctionName())); if (column_id == 0) new_node->getArguments().getNodes() = {lhs, rhs}; else if (column_id == 1) @@ -93,7 +94,12 @@ public: if (!new_node) return; - node = new_node; + QueryTreeNodePtr res = std::move(new_node); + + if (!res->getResultType()->equals(*function_node->getResultType())) + res = createCastFunction(res, function_node->getResultType(), getContext()); + + node = std::move(res); } diff --git a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference index 43282d09bab..5028adae32e 100644 --- a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference +++ b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference @@ -1,2 +1,3 @@ 4 2 100 99 +(3.22,1) diff --git a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql index 5eba14ea528..8491018eb72 100644 --- a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql +++ b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql @@ -14,3 +14,5 @@ INSERT INTO test_table VALUES (1, 1); SELECT sum((2 * id) as func), func FROM test_table GROUP BY id; SELECT max(100-number), min(100-number) FROM numbers(2); + +select (sum(toDecimal64(2.11, 15) - number), 1) FROM numbers(2); From 19b90d8348aec4dc4fc038c89155a169ba4d6224 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 28 Mar 2024 16:28:59 +0100 Subject: [PATCH 0747/1165] Resolve conflicts after merge, add system.filesystem_cache_settings --- src/Interpreters/Cache/EvictionCandidates.cpp | 23 +++++- src/Interpreters/Cache/EvictionCandidates.h | 14 ++++ src/Interpreters/Cache/FileCache.cpp | 23 ++++-- .../Cache/LRUFileCachePriority.cpp | 71 +++++++++--------- src/Interpreters/Cache/Metadata.cpp | 2 +- .../Cache/SLRUFileCachePriority.cpp | 2 +- .../StorageSystemFilesystemCacheSettings.cpp | 72 +++++++++++++++++++ .../StorageSystemFilesystemCacheSettings.h | 22 ++++++ src/Storages/System/attachSystemTables.cpp | 2 + ...resize_filesystem_cache_hardcore.reference | 7 ++ ...ically_resize_filesystem_cache_hardcore.sh | 46 ++++++++++++ 11 files changed, 237 insertions(+), 47 deletions(-) create mode 100644 src/Storages/System/StorageSystemFilesystemCacheSettings.cpp create mode 100644 src/Storages/System/StorageSystemFilesystemCacheSettings.h create mode 100644 tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.reference create mode 100755 tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.sh diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index cb15af72704..080ef675918 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -57,13 +57,25 @@ void EvictionCandidates::add( ++candidates_size; } +void EvictionCandidates::removeQueueEntries(const CachePriorityGuard::Lock & lock) +{ + for (const auto & [key, key_candidates] : candidates) + { + for (const auto & candidate : key_candidates.candidates) + candidate->getQueueIterator()->remove(lock); + } + invalidated_queue_entries = true; +} + void EvictionCandidates::evict() { if (candidates.empty()) return; auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::FilesystemCacheEvictMicroseconds); - queue_entries_to_invalidate.reserve(candidates_size); + + if (!invalidated_queue_entries) + queue_entries_to_invalidate.reserve(candidates_size); for (auto & [key, key_candidates] : candidates) { @@ -111,7 +123,9 @@ void EvictionCandidates::evict() /// it was freed in favour of some reserver, so we can make it visibly /// free only for that particular reserver. - queue_entries_to_invalidate.push_back(iterator); + if (!invalidated_queue_entries) + queue_entries_to_invalidate.push_back(iterator); + key_candidates.candidates.pop_back(); } } @@ -165,4 +179,9 @@ void EvictionCandidates::setSpaceHolder( hold_space = std::make_unique(size, elements, priority, lock); } +void EvictionCandidates::insert(EvictionCandidates && other, const CachePriorityGuard::Lock &) +{ + candidates.insert(make_move_iterator(other.candidates.begin()), make_move_iterator(other.candidates.end())); +} + } diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index 2745d508a5d..571010a14bc 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -9,6 +9,13 @@ class EvictionCandidates public: using FinalizeEvictionFunc = std::function; + EvictionCandidates() = default; + EvictionCandidates(EvictionCandidates && other) noexcept + { + candidates = std::move(other.candidates); + candidates_size = std::move(other.candidates_size); + queue_entries_to_invalidate = std::move(other.queue_entries_to_invalidate); + } ~EvictionCandidates(); void add( @@ -16,8 +23,12 @@ public: LockedKey & locked_key, const CachePriorityGuard::Lock &); + void insert(EvictionCandidates && other, const CachePriorityGuard::Lock &); + void evict(); + void removeQueueEntries(const CachePriorityGuard::Lock &); + void onFinalize(FinalizeEvictionFunc && func) { on_finalize.emplace_back(std::move(func)); } void finalize( @@ -47,7 +58,10 @@ private: size_t candidates_size = 0; std::vector on_finalize; + std::vector queue_entries_to_invalidate; + bool invalidated_queue_entries = false; + IFileCachePriority::HoldSpacePtr hold_space; }; diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index e845559e8a7..75e199c544b 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1383,7 +1383,7 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, if (new_settings.max_size != actual_settings.max_size || new_settings.max_elements != actual_settings.max_elements) { - std::vector evicted_paths; + std::optional eviction_candidates; { cache_is_being_resized.store(true, std::memory_order_relaxed); SCOPE_EXIT({ @@ -1391,18 +1391,27 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, }); auto cache_lock = lockCache(); - FileCacheReserveStat stat; - auto eviction_candidates = main_priority->collectCandidatesForEviction( - new_settings.max_size, new_settings.max_elements, 0/* max_candidates_to_evict */, stat, cache_lock); - evicted_paths = eviction_candidates.evictFromMemory(nullptr, cache_lock); + FileCacheReserveStat stat; + eviction_candidates.emplace(main_priority->collectCandidatesForEviction( + new_settings.max_size, new_settings.max_elements, 0/* max_candidates_to_evict */, stat, cache_lock)); + + eviction_candidates->removeQueueEntries(cache_lock); main_priority->modifySizeLimits( new_settings.max_size, new_settings.max_elements, new_settings.slru_size_ratio, cache_lock); } - for (const auto & path : evicted_paths) - fs::remove(path); + try + { + eviction_candidates->evict(); + } + catch (...) + { + auto cache_lock = lockCache(); + eviction_candidates->finalize(nullptr, cache_lock); + throw; + } LOG_INFO(log, "Changed max_size from {} to {}, max_elements from {} to {}", actual_settings.max_size, new_settings.max_size, diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 4a80bce7658..d1c46691c37 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -279,10 +279,42 @@ bool LRUFileCachePriority::collectCandidatesForEviction( auto can_fit = [&] { - return canFit(size, 1, stat.stat.releasable_size, stat.stat.releasable_count, lock); + return canFit(size, 1, stat.total_stat.releasable_size, stat.total_stat.releasable_count, lock); }; + iterateForEviction(res, stat, can_fit, lock); - return can_fit(); + if (can_fit()) + { + /// As eviction is done without a cache priority lock, + /// then if some space was partially available and some needed + /// to be freed via eviction, we need to make sure that this + /// partially available space is still available + /// after we finish with eviction for non-available space. + /// So we create a space holder for the currently available part + /// of the required space for the duration of eviction of the other + /// currently non-available part of the space. + + const size_t hold_size = size > stat.total_stat.releasable_size + ? size - stat.total_stat.releasable_size + : 0; + + const size_t hold_elements = elements > stat.total_stat.releasable_count + ? elements - stat.total_stat.releasable_count + : 0; + + if (hold_size || hold_elements) + res.setSpaceHolder(hold_size, hold_elements, *this, lock); + + // LOG_TEST(log, "Collected {} candidates for eviction (total size: {}). " + // "Took hold of size {} and elements {}", + // res.size(), stat.total_stat.releasable_size, hold_size, hold_elements); + + return true; + } + else + { + return false; + } } EvictionCandidates LRUFileCachePriority::collectCandidatesForEviction( @@ -295,7 +327,7 @@ EvictionCandidates LRUFileCachePriority::collectCandidatesForEviction( EvictionCandidates res; auto stop_condition = [&, this]() { - return canFit(0, 0, stat.stat.releasable_size, stat.stat.releasable_count, + return canFit(0, 0, stat.total_stat.releasable_size, stat.total_stat.releasable_count, lock, &desired_size, &desired_elements_count) || (max_candidates_to_evict && res.size() >= max_candidates_to_evict); }; @@ -334,39 +366,6 @@ void LRUFileCachePriority::iterateForEviction( { return stop_condition() ? IterationResult::BREAK : iterate_func(locked_key, segment_metadata); }, lock); - - if (can_fit()) - { - /// As eviction is done without a cache priority lock, - /// then if some space was partially available and some needed - /// to be freed via eviction, we need to make sure that this - /// partially available space is still available - /// after we finish with eviction for non-available space. - /// So we create a space holder for the currently available part - /// of the required space for the duration of eviction of the other - /// currently non-available part of the space. - - const size_t hold_size = size > stat.total_stat.releasable_size - ? size - stat.total_stat.releasable_size - : 0; - - const size_t hold_elements = elements > stat.total_stat.releasable_count - ? elements - stat.total_stat.releasable_count - : 0; - - if (hold_size || hold_elements) - res.setSpaceHolder(hold_size, hold_elements, *this, lock); - - // LOG_TEST(log, "Collected {} candidates for eviction (total size: {}). " - // "Took hold of size {} and elements {}", - // res.size(), stat.total_stat.releasable_size, hold_size, hold_elements); - - return true; - } - else - { - return false; - } } LRUFileCachePriority::LRUIterator LRUFileCachePriority::move( diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 082c33032f2..26611f02379 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -941,7 +941,7 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl( file_segment->detach(segment_lock, *this); - if (!remove_only_metadata) + // if (!remove_only_metadata) { try { diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index df4d4276e89..79ca489cea2 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -263,7 +263,7 @@ EvictionCandidates SLRUFileCachePriority::collectCandidatesForEviction( desired_probationary_size, desired_probationary_elements_num, max_candidates_to_evict, stat, lock); chassert(!max_candidates_to_evict || res.size() <= max_candidates_to_evict); - chassert(res.size() == stat.stat.releasable_count); + chassert(res.size() == stat.total_stat.releasable_count); if (max_candidates_to_evict && res.size() == max_candidates_to_evict) return res; diff --git a/src/Storages/System/StorageSystemFilesystemCacheSettings.cpp b/src/Storages/System/StorageSystemFilesystemCacheSettings.cpp new file mode 100644 index 00000000000..8915032baf7 --- /dev/null +++ b/src/Storages/System/StorageSystemFilesystemCacheSettings.cpp @@ -0,0 +1,72 @@ +#include "StorageSystemFilesystemCacheSettings.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +ColumnsDescription StorageSystemFilesystemCacheSettings::getColumnsDescription() +{ + return ColumnsDescription + { + {"cache_name", std::make_shared(), "Name of the cache object"}, + {"path", std::make_shared(), "Cache directory"}, + {"max_size", std::make_shared(), "Cache size limit by the number of bytes"}, + {"max_elements", std::make_shared(), "Cache size limit by the number of elements"}, + {"current_size", std::make_shared(), "Current cache size by the number of bytes"}, + {"current_elements", std::make_shared(), "Current cache size by the number of elements"}, + {"max_file_segment_size", std::make_shared(), "Maximum allowed file segment size"}, + {"boundary_alignment", std::make_shared(), "Boundary alignment of file segments"}, + {"cache_on_write_operations", std::make_shared(), "Write-through cache enablemenet setting"}, + {"cache_hits_threshold", std::make_shared(), "Cache hits threshold enablemenet setting"}, + {"background_download_threads", std::make_shared(), "Number of background download threads"}, + {"background_download_queue_size_limit", std::make_shared(), "Queue size limit for background download"}, + {"load_metadata_threads", std::make_shared(), "Number of load metadata threads"}, + {"enable_bypass_cache_threshold", std::make_shared(), "Bypass cache threshold limit enablement setting"}, + }; +} + +StorageSystemFilesystemCacheSettings::StorageSystemFilesystemCacheSettings(const StorageID & table_id_) + : IStorageSystemOneBlock(table_id_, getColumnsDescription()) +{ +} + +void StorageSystemFilesystemCacheSettings::fillData( + MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector) const +{ + context->checkAccess(AccessType::SHOW_FILESYSTEM_CACHES); + + auto caches = FileCacheFactory::instance().getAll(); + + for (const auto & [cache_name, cache_data] : caches) + { + const auto & settings = cache_data->getSettings(); + const auto & cache = cache_data->cache; + + size_t i = 0; + res_columns[i++]->insert(cache_name); + res_columns[i++]->insert(settings.base_path); + res_columns[i++]->insert(settings.max_size); + res_columns[i++]->insert(settings.max_elements); + res_columns[i++]->insert(cache->getUsedCacheSize()); + res_columns[i++]->insert(cache->getFileSegmentsNum()); + res_columns[i++]->insert(settings.max_file_segment_size); + res_columns[i++]->insert(settings.boundary_alignment); + res_columns[i++]->insert(settings.cache_on_write_operations); + res_columns[i++]->insert(settings.cache_hits_threshold); + res_columns[i++]->insert(settings.background_download_threads); + res_columns[i++]->insert(settings.background_download_queue_size_limit); + res_columns[i++]->insert(settings.load_metadata_threads); + res_columns[i++]->insert(settings.enable_bypass_cache_with_threshold); + } +} + +} diff --git a/src/Storages/System/StorageSystemFilesystemCacheSettings.h b/src/Storages/System/StorageSystemFilesystemCacheSettings.h new file mode 100644 index 00000000000..59a123c32c1 --- /dev/null +++ b/src/Storages/System/StorageSystemFilesystemCacheSettings.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class StorageSystemFilesystemCacheSettings final : public IStorageSystemOneBlock +{ +public: + explicit StorageSystemFilesystemCacheSettings(const StorageID & table_id_); + + std::string getName() const override { return "SystemFilesystemCacheSettings"; } + + static ColumnsDescription getColumnsDescription(); + +protected: + void fillData(MutableColumns & res_columns, ContextPtr, const ActionsDAG::Node *, std::vector) const override; +}; + +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index cd8be60e342..6ff86b26ca9 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -78,6 +78,7 @@ #include #include #include +#include #include #include #include @@ -213,6 +214,7 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b attach(context, system_database, "part_moves_between_shards", "Contains information about parts which are currently in a process of moving between shards and their progress."); attach(context, system_database, "asynchronous_inserts", "Contains information about pending asynchronous inserts in queue in server's memory."); attachNoDescription(context, system_database, "filesystem_cache", "Contains information about all entries inside filesystem cache for remote objects."); + attachNoDescription(context, system_database, "filesystem_cache_settings", "Contains information about all filesystem cache settings"); attachNoDescription(context, system_database, "query_cache", "Contains information about all entries inside query cache in server's memory."); attachNoDescription(context, system_database, "remote_data_paths", "Contains a mapping from a filename on local filesystem to a blob name inside object storage."); attach(context, system_database, "certificates", "Contains information about available certificates and their sources."); diff --git a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.reference b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.reference new file mode 100644 index 00000000000..fcb49fa9945 --- /dev/null +++ b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.reference @@ -0,0 +1,7 @@ +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.sh b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.sh new file mode 100755 index 00000000000..79c43048b89 --- /dev/null +++ b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-s3-storage, no-random-settings + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +disk_name="s3_cache" + +$CLICKHOUSE_CLIENT -nm --query " +DROP TABLE IF EXISTS test; +CREATE TABLE test (a String) engine=MergeTree() ORDER BY tuple() SETTINGS disk = '$disk_name'; +INSERT INTO test SELECT randomString(1000); +" + +$CLICKHOUSE_CLIENT --query "SELECT * FROM test FORMAT Null" + +prev_max_size=$($CLICKHOUSE_CLIENT --query "SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '$disk_name'") +$CLICKHOUSE_CLIENT --query "SELECT current_size > 0 FROM system.filesystem_cache_settings WHERE cache_name = '$disk_name' FORMAT TabSeparated" + +config_path=/etc/clickhouse-server/config.d/storage_conf.xml +config_path_tmp=$config_path.tmp + +new_max_size=$($CLICKHOUSE_CLIENT --query "SELECT divide(max_size, 2) FROM system.filesystem_cache_settings WHERE cache_name = '$disk_name'") +sed -i "s|$prev_max_size<\/max_size>|$new_max_size<\/max_size>|" $config_path + +# echo $prev_max_size +# echo $new_max_size + +$CLICKHOUSE_CLIENT -nm --query " +set send_logs_level='fatal'; +SYSTEM RELOAD CONFIG" + +$CLICKHOUSE_CLIENT --query "SELECT max_size == $new_max_size FROM system.filesystem_cache_settings WHERE cache_name = '$disk_name' FORMAT TabSeparated" +$CLICKHOUSE_CLIENT --query "SELECT current_size > 0 FROM system.filesystem_cache_settings WHERE cache_name = '$disk_name' FORMAT TabSeparated" +$CLICKHOUSE_CLIENT --query "SELECT current_size <= max_size FROM system.filesystem_cache_settings WHERE cache_name = '$disk_name' FORMAT TabSeparated" + +sed -i "s|$new_max_size<\/max_size>|$prev_max_size<\/max_size>|" $config_path + +$CLICKHOUSE_CLIENT -nm --query " +set send_logs_level='fatal'; +SYSTEM RELOAD CONFIG" + +$CLICKHOUSE_CLIENT --query "SELECT max_size == $prev_max_size FROM system.filesystem_cache_settings WHERE cache_name = '$disk_name' FORMAT TabSeparated" +$CLICKHOUSE_CLIENT --query "SELECT current_size > 0 FROM system.filesystem_cache_settings WHERE cache_name = '$disk_name' FORMAT TabSeparated" +$CLICKHOUSE_CLIENT --query "SELECT current_size <= max_size FROM system.filesystem_cache_settings WHERE cache_name = '$disk_name' FORMAT TabSeparated" From 56e95c84c17c9201c35af71408f0684683d74f32 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 28 Mar 2024 16:45:02 +0100 Subject: [PATCH 0748/1165] Update DatabaseOnDisk.cpp --- src/Databases/DatabaseOnDisk.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 550f1a756cb..d8acfb5fa01 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -109,7 +109,7 @@ std::pair createTableFromAST( } else { - columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, LoadingStrictnessLevel::ATTACH); + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, mode); constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints); } } From 0b85d8570bf58e193f1df41a38f2b82be5ac96a6 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 28 Mar 2024 16:46:44 +0100 Subject: [PATCH 0749/1165] Resubmit 'Update invalidate_query_response on dictionary startup' --- src/Interpreters/ExternalLoader.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index bd56a540128..73dacfacf79 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -996,6 +996,14 @@ private: if (!new_object && !new_exception) throw Exception(ErrorCodes::LOGICAL_ERROR, "No object created and no exception raised for {}", type_name); + if (!info->object && new_object) + { + /// If we loaded the object for the first time then we should set `invalidate_query_response` to the current value. + /// Otherwise we will immediately try to reload the object again despite the fact that it was just loaded. + bool is_modified = new_object->isModified(); + LOG_TRACE(log, "Object '{}' was{} modified", name, (is_modified ? "" : " not")); + } + /// Saving the result of the loading. { LoadingGuardForAsyncLoad lock(async, mutex); From 1d453af6ff2f49fb2dc3beabbcdc2e00fe21757f Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 28 Mar 2024 16:47:22 +0100 Subject: [PATCH 0750/1165] Fix style check --- src/Interpreters/Cache/LRUFileCachePriority.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index d1c46691c37..78ece5a3124 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -16,9 +16,6 @@ namespace ProfileEvents { extern const Event FilesystemCacheEvictionSkippedFileSegments; extern const Event FilesystemCacheEvictionTries; - extern const Event FilesystemCacheEvictMicroseconds; - extern const Event FilesystemCacheEvictedBytes; - extern const Event FilesystemCacheEvictedFileSegments; extern const Event FilesystemCacheEvictionSkippedEvictingFileSegments; } From e2b0fde0f077b8961e67a51f2f0c8ff2fe12ab74 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 28 Mar 2024 17:22:14 +0100 Subject: [PATCH 0751/1165] Remove unused setting --- src/Storages/StorageMemory.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 2de2878002a..013c9f66c2c 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -46,7 +46,6 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int CANNOT_RESTORE_TABLE; extern const int NOT_IMPLEMENTED; - extern const int SETTING_CONSTRAINT_VIOLATION; } class MemorySink : public SinkToStorage From c834a11b653e9a67abe990d9db24dfa998aa9aa5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Mar 2024 16:32:12 +0000 Subject: [PATCH 0752/1165] Ignore IfChainToMultiIfPass if returned type changed. --- src/Analyzer/Passes/IfChainToMultiIfPass.cpp | 6 ++++++ .../0_stateless/01388_multi_if_optimization.reference | 2 ++ tests/queries/0_stateless/01388_multi_if_optimization.sql | 3 +++ 3 files changed, 11 insertions(+) diff --git a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp index 70b717f3108..beb2247607e 100644 --- a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp +++ b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp @@ -65,6 +65,12 @@ public: auto multi_if_function = std::make_shared("multiIf"); multi_if_function->getArguments().getNodes() = std::move(multi_if_arguments); multi_if_function->resolveAsFunction(multi_if_function_ptr->build(multi_if_function->getArgumentColumns())); + + /// Ignore if returned type changed. + /// Example : SELECT now64(if(Null, NULL, if(Null, nan, toFloat64(number))), Null) FROM numbers(2) + if (!multi_if_function->getResultType()->equals(*function_node->getResultType())) + return; + node = std::move(multi_if_function); } diff --git a/tests/queries/0_stateless/01388_multi_if_optimization.reference b/tests/queries/0_stateless/01388_multi_if_optimization.reference index 6dbe0f0d96f..9c2990274fe 100644 --- a/tests/queries/0_stateless/01388_multi_if_optimization.reference +++ b/tests/queries/0_stateless/01388_multi_if_optimization.reference @@ -2,3 +2,5 @@ SELECT if(number = 1, \'hello\', if(number = 2, \'world\', \'xyz\')) FROM numbers(10) SELECT multiIf(number = 1, \'hello\', number = 2, \'world\', \'xyz\') FROM numbers(10) +\N +\N diff --git a/tests/queries/0_stateless/01388_multi_if_optimization.sql b/tests/queries/0_stateless/01388_multi_if_optimization.sql index 345fcfb6fcc..0ad6df9fdfa 100644 --- a/tests/queries/0_stateless/01388_multi_if_optimization.sql +++ b/tests/queries/0_stateless/01388_multi_if_optimization.sql @@ -3,3 +3,6 @@ SET optimize_if_chain_to_multiif = 0; EXPLAIN SYNTAX SELECT number = 1 ? 'hello' : (number = 2 ? 'world' : 'xyz') FROM numbers(10); SET optimize_if_chain_to_multiif = 1; EXPLAIN SYNTAX SELECT number = 1 ? 'hello' : (number = 2 ? 'world' : 'xyz') FROM numbers(10); + +-- fuzzed +SELECT now64(if(Null, NULL, if(Null, nan, toFloat64(number))), Null) FROM numbers(2); From be3c498833566f20e47afe1dc65fdf58af9d7ee1 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Thu, 28 Mar 2024 18:22:26 +0100 Subject: [PATCH 0753/1165] Support more that 255 replicas in system table --- src/Storages/System/StorageSystemReplicas.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 35550de11cb..e4b8d78a3a6 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -235,8 +235,8 @@ StorageSystemReplicas::StorageSystemReplicas(const StorageID & table_id_) "If log_pointer is much smaller than log_max_index, something is wrong."}, { "last_queue_update", std::make_shared(), "When the queue was updated last time."}, { "absolute_delay", std::make_shared(), "How big lag in seconds the current replica has."}, - { "total_replicas", std::make_shared(), "The total number of known replicas of this table."}, - { "active_replicas", std::make_shared(), "The number of replicas of this table that have a session in ClickHouse Keeper (i.e., the number of functioning replicas)."}, + { "total_replicas", std::make_shared(), "The total number of known replicas of this table."}, + { "active_replicas", std::make_shared(), "The number of replicas of this table that have a session in ClickHouse Keeper (i.e., the number of functioning replicas)."}, { "lost_part_count", std::make_shared(), "The number of data parts lost in the table by all replicas in total since table creation. Value is persisted in ClickHouse Keeper and can only increase."}, { "last_queue_update_exception", std::make_shared(), "When the queue contains broken entries. Especially important when ClickHouse breaks backward compatibility between versions and log entries written by newer versions aren't parseable by old versions."}, { "zookeeper_exception", std::make_shared(), "The last exception message, got if the error happened when fetching the info from ClickHouse Keeper."}, From a94143ee194176ead64603f8be9d54c8548fc6d8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Mar 2024 18:26:03 +0100 Subject: [PATCH 0754/1165] Fix stress tests for analyzer due to experimental WINDOW VIEW (by disabling it) CI found [1]: 2024.03.28 13:16:00.797344 [ 99740 ] {} Application: Caught exception while loading metadata: Code: 695. DB::Exception: Load job 'load table 01069_window_view_proc_tumble_watch.wv' failed: Code: 1. DB::Exception: Experimental WINDOW VIEW feature is not supported with new infrastructure for query analysis (the setting 'allow_experimental_analyzer'): Cannot attach table `01069_window_view_proc_tumble_watch`.`wv` from metadata file /var/lib/clickhouse/store/dd8/dd82a4f5-5485-4747-9172-8976d1194c54/wv.sql from query ATTACH WINDOW VIEW `01069_window_view_proc_tumble_watch`.wv UUID '35cc7d27-df3b-4569-9cb9-2ccf6cb1ff4c' (`count` UInt64) ENGINE = Memory AS SELECT count(a) AS count FROM `01069_window_view_proc_tumble_watch`.mt GROUP BY tumble(timestamp, toIntervalSecond('1'), 'US/Samoa') AS wid. (UNSUPPORTED_METHOD), Stack trace (when copying this message, always include the lines below): [1]: https://s3.amazonaws.com/clickhouse-test-reports/61997/ced095394b6fb6d50ed8b6834bd5911ad4702c6e/stateless_tests__tsan__s3_storage__[5_5].html Signed-off-by: Azat Khuzhin --- docker/test/stress/run.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index cd7c1243a4a..6c6caf872e9 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -87,6 +87,25 @@ if [ "$cache_policy" = "SLRU" ]; then mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml fi +# Disable experimental WINDOW VIEW tests for stress tests, since they may be +# created with old analyzer and then, after server restart it will refuse to +# start. +# FIXME: remove once the support for WINDOW VIEW will be implemented in analyzer. +sudo cat /etc/clickhouse-server/users.d/stress_tests_overrides.xml < + + + false + + + + + + + + +EOL + start_server clickhouse-client --query "SHOW TABLES FROM datasets" From 54d7a6041e36d6d8eaf793f0a931e34b6bd8e103 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Mar 2024 17:35:38 +0000 Subject: [PATCH 0755/1165] Fix type for ConvertInToEqualPass --- src/Analyzer/Passes/ConvertInToEqualPass.cpp | 5 ++++- tests/queries/0_stateless/03013_optimize_in_to_equal.sql | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp index 66a37fea5bd..6980983486a 100644 --- a/src/Analyzer/Passes/ConvertInToEqualPass.cpp +++ b/src/Analyzer/Passes/ConvertInToEqualPass.cpp @@ -40,7 +40,10 @@ public: return ; auto result_func_name = MAPPING.at(func_node->getFunctionName()); auto equal = std::make_shared(result_func_name); - QueryTreeNodes arguments{column_node->clone(), constant_node->clone()}; + QueryTreeNodes arguments{ + column_node->clone(), + std::make_shared(constant_node->getValue(), removeNullable(constant_node->getResultType())) + }; equal->getArguments().getNodes() = std::move(arguments); FunctionOverloadResolverPtr resolver; bool decimal_check_overflow = getContext()->getSettingsRef().decimal_check_overflow; diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.sql b/tests/queries/0_stateless/03013_optimize_in_to_equal.sql index ba6eb5d4f5f..e0eaa84cb8d 100644 --- a/tests/queries/0_stateless/03013_optimize_in_to_equal.sql +++ b/tests/queries/0_stateless/03013_optimize_in_to_equal.sql @@ -27,3 +27,6 @@ select '-------------------'; explain query tree select * from test where x not in (NULL); select '-------------------'; explain query tree select * from test where x in (NULL); +select '-------------------'; +--- fuzzed +SELECT number FROM numbers(2) WHERE arrayExists(_ -> (_ IN toNullable(4294967290)), [number]); From 8758d7f3a8ebbdb4fa0bb071e4f23695fc53215e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Mar 2024 18:19:40 +0000 Subject: [PATCH 0756/1165] Fixing test. --- src/Analyzer/Passes/ConvertInToEqualPass.cpp | 7 +++---- .../0_stateless/03013_optimize_in_to_equal.reference | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp index 6980983486a..b204d2fb922 100644 --- a/src/Analyzer/Passes/ConvertInToEqualPass.cpp +++ b/src/Analyzer/Passes/ConvertInToEqualPass.cpp @@ -40,10 +40,9 @@ public: return ; auto result_func_name = MAPPING.at(func_node->getFunctionName()); auto equal = std::make_shared(result_func_name); - QueryTreeNodes arguments{ - column_node->clone(), - std::make_shared(constant_node->getValue(), removeNullable(constant_node->getResultType())) - }; + auto new_const = std::make_shared(constant_node->getValue(), removeNullable(constant_node->getResultType())); + new_const->getSourceExpression() = constant_node->getSourceExpression(); + QueryTreeNodes arguments{column_node->clone(), new_const}; equal->getArguments().getNodes() = std::move(arguments); FunctionOverloadResolverPtr resolver; bool decimal_check_overflow = getContext()->getSettingsRef().decimal_check_overflow; diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.reference b/tests/queries/0_stateless/03013_optimize_in_to_equal.reference index 93ac91bd957..7d1118b7730 100644 --- a/tests/queries/0_stateless/03013_optimize_in_to_equal.reference +++ b/tests/queries/0_stateless/03013_optimize_in_to_equal.reference @@ -186,3 +186,4 @@ QUERY id: 0 LIST id: 6, nodes: 2 COLUMN id: 7, column_name: x, result_type: String, source_id: 3 CONSTANT id: 8, constant_value: NULL, constant_value_type: Nullable(Nothing) +------------------- From 7fb4ace6ef8a39c826e6726675015b6ab3fd2391 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 28 Mar 2024 19:24:10 +0100 Subject: [PATCH 0757/1165] clang fix --- src/Storages/StorageMemory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 013c9f66c2c..055d9ee6fa1 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -105,7 +105,7 @@ public: auto new_data = std::make_unique(*(storage.data.get())); UInt64 new_total_rows = storage.total_size_rows.load(std::memory_order_relaxed) + inserted_rows; UInt64 new_total_bytes = storage.total_size_bytes.load(std::memory_order_relaxed) + inserted_bytes; - auto & memory_settings = storage.getMemorySettingsRef(); + const auto & memory_settings = storage.getMemorySettingsRef(); while (!new_data->empty() && ((memory_settings.max_bytes_to_keep && new_total_bytes > memory_settings.max_bytes_to_keep) || (memory_settings.max_rows_to_keep && new_total_rows > memory_settings.max_rows_to_keep))) From fd01a1227699f786d9c977cfda3512ed0fef33ef Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 Mar 2024 18:26:15 +0000 Subject: [PATCH 0758/1165] Fixing another case with grouping set --- .../Passes/GroupingFunctionsResolvePass.cpp | 2 +- src/Processors/QueryPlan/AggregatingStep.cpp | 26 +++++++++++-------- ...up_by_use_nulls_analyzer_crashes.reference | 10 +++++++ ...23_group_by_use_nulls_analyzer_crashes.sql | 2 ++ 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp index cf0fb824b4d..085519c7220 100644 --- a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp @@ -146,7 +146,7 @@ void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context) if (query_node_typed.hasGroupBy()) { /// It is expected by execution layer that if there are only 1 grouping set it will be removed - if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.getGroupBy().getNodes().size() == 1) + if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.getGroupBy().getNodes().size() == 1 && !context->getSettingsRef().group_by_use_nulls) { auto grouping_set_list_node = query_node_typed.getGroupBy().getNodes().front(); auto & grouping_set_list_node_typed = grouping_set_list_node->as(); diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index a76bacdd97b..74f293e5682 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -191,20 +191,24 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B const size_t streams = pipeline.getNumStreams(); auto input_header = pipeline.getHeader(); - pipeline.transform([&](OutputPortRawPtrs ports) + + if (grouping_sets_size > 1) { - Processors copiers; - copiers.reserve(ports.size()); - - for (auto * port : ports) + pipeline.transform([&](OutputPortRawPtrs ports) { - auto copier = std::make_shared(input_header, grouping_sets_size); - connect(*port, copier->getInputPort()); - copiers.push_back(copier); - } + Processors copiers; + copiers.reserve(ports.size()); - return copiers; - }); + for (auto * port : ports) + { + auto copier = std::make_shared(input_header, grouping_sets_size); + connect(*port, copier->getInputPort()); + copiers.push_back(copier); + } + + return copiers; + }); + } pipeline.transform([&](OutputPortRawPtrs ports) { diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference index 9d8381407fb..4081b82a8f5 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference @@ -50,3 +50,13 @@ ([7]) ([8]) ([9]) +(0) +(1) +(2) +(3) +(4) +(5) +(6) +(7) +(8) +(9) diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql index 670c4fb1284..012da547581 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql @@ -3,3 +3,5 @@ set allow_experimental_analyzer = 1, group_by_use_nulls = 1; SELECT tuple(tuple(number)) as x FROM numbers(10) GROUP BY (number, tuple(number)) with cube order by x; select tuple(array(number)) as x FROM numbers(10) GROUP BY number, array(number) WITH ROLLUP order by x; + +SELECT tuple(number) AS x FROM numbers(10) GROUP BY GROUPING SETS (number) order by x; From 1ea058a702b68b9d3342ed01a700341b584c5979 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 28 Mar 2024 19:33:23 +0100 Subject: [PATCH 0759/1165] ReadWriteBufferFromHTTP set right header host when redireced --- src/Common/HTTPConnectionPool.cpp | 12 +++++++++--- src/IO/ReadWriteBufferFromHTTP.cpp | 29 ++++++++++++++++------------- src/IO/ReadWriteBufferFromHTTP.h | 1 - 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index f729b8ea8d0..cd2505df7f3 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -295,8 +295,13 @@ private: String getTarget() const { if (!Session::getProxyConfig().host.empty()) - return fmt::format("{} over proxy {}", Session::getHost(), Session::getProxyConfig().host); - return Session::getHost(); + return fmt::format("{}:{} over proxy {}", + Session::getHost(), + Session::getPort(), + Session::getProxyConfig().host); + return fmt::format("{}:{}", + Session::getHost(), + Session::getPort()); } void flushRequest() override @@ -472,7 +477,8 @@ public: String getTarget() const { if (!proxy_configuration.isEmpty()) - return fmt::format("{} over proxy {}", host, proxy_configuration.host); + return fmt::format("{} over proxy {}", + host, proxy_configuration.host); return host; } diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp index 61c40c5a8fc..63a91026701 100644 --- a/src/IO/ReadWriteBufferFromHTTP.cpp +++ b/src/IO/ReadWriteBufferFromHTTP.cpp @@ -97,7 +97,7 @@ size_t ReadWriteBufferFromHTTP::getOffset() const void ReadWriteBufferFromHTTP::prepareRequest(Poco::Net::HTTPRequest & request, std::optional range) const { - request.setHost(initial_uri.getHost()); // use original, not resolved host name in header + request.setHost(current_uri.getHost()); if (out_stream_callback) request.setChunkedTransferEncoding(true); @@ -237,15 +237,15 @@ ReadWriteBufferFromHTTP::ReadWriteBufferFromHTTP( } ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callImpl( - Poco::Net::HTTPResponse & response, const Poco::URI & uri_, const std::string & method_, const std::optional & range, bool allow_redirects) const + Poco::Net::HTTPResponse & response, const std::string & method_, const std::optional & range, bool allow_redirects) const { if (remote_host_filter) - remote_host_filter->checkURL(uri_); + remote_host_filter->checkURL(current_uri); - Poco::Net::HTTPRequest request(method_, uri_.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1); + Poco::Net::HTTPRequest request(method_, current_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1); prepareRequest(request, range); - auto session = makeHTTPSession(connection_group, uri_, timeouts, proxy_config); + auto session = makeHTTPSession(connection_group, current_uri, timeouts, proxy_config); ProfileEvents::increment(ProfileEvents::ReadWriteBufferFromHTTPRequestsSent); @@ -263,7 +263,7 @@ ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callImpl( ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callWithRedirects( Poco::Net::HTTPResponse & response, const String & method_, const std::optional & range) { - auto result = callImpl(response, current_uri, method_, range, true); + auto result = callImpl(response, method_, range, true); while (isRedirect(response.getStatus())) { @@ -279,8 +279,7 @@ ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callWithRedirects( initial_uri.toString(), max_redirects ? "increase the allowed maximum number of" : "allow"); current_uri = uri_redirect; - - result = callImpl(response, uri_redirect, method_, range, true); + result = callImpl(response, method_, range, true); } return result; @@ -347,9 +346,11 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, { if (!mute_logging) LOG_ERROR(log, - "Failed to make request to '{}'. Error: '{}'. " + "Failed to make request to `{}`{}. " + "Error: '{}'. " "Failed at try {}/{}.", - initial_uri.toString(), error_message, + initial_uri.toString(), current_uri == initial_uri ? String() : fmt::format(" redirect to `{}`", current_uri.toString()), + error_message, attempt, read_settings.http_max_tries); std::rethrow_exception(exception); @@ -361,10 +362,12 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, if (!mute_logging) LOG_INFO(log, - "Failed to make request to `{}`. Error: {}. " + "Failed to make request to `{}`{}. " + "Error: {}. " "Failed at try {}/{}. " "Will retry with current backoff wait is {}/{} ms.", - initial_uri.toString(), error_message, + initial_uri.toString(), current_uri == initial_uri ? String() : fmt::format(" redirect to `{}`", current_uri.toString()), + error_message, attempt + 1, read_settings.http_max_tries, milliseconds_to_wait, read_settings.http_retry_max_backoff_ms); @@ -512,7 +515,7 @@ size_t ReadWriteBufferFromHTTP::readBigAt(char * to, size_t n, size_t offset, co auto range = HTTPRange{offset, offset + n - 1}; Poco::Net::HTTPResponse response; - auto result = callImpl(response, current_uri, method, range, false); + auto result = callImpl(response, method, range, false); if (response.getStatus() != Poco::Net::HTTPResponse::HTTPStatus::HTTP_PARTIAL_CONTENT && (offset != 0 || offset + n < *file_info->file_size)) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 224aac809a4..f496fe3ddcd 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -107,7 +107,6 @@ private: CallResult callImpl( Poco::Net::HTTPResponse & response, - const Poco::URI & uri_, const std::string & method_, const std::optional & range, bool allow_redirects) const; From d967a9280c9ef2b182be441c459a9f8151141ed2 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Thu, 28 Mar 2024 19:41:30 +0100 Subject: [PATCH 0760/1165] Update test --- .../0_stateless/02117_show_create_table_system.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index c9638e62655..bdd0da7d166 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -881,8 +881,8 @@ CREATE TABLE system.replicas `log_pointer` UInt64, `last_queue_update` DateTime, `absolute_delay` UInt64, - `total_replicas` UInt8, - `active_replicas` UInt8, + `total_replicas` UInt32, + `active_replicas` UInt32, `lost_part_count` UInt64, `last_queue_update_exception` String, `zookeeper_exception` String, From ef826ee881b2c260999845f86820389c4378e2ab Mon Sep 17 00:00:00 2001 From: Blargian Date: Thu, 28 Mar 2024 19:47:50 +0100 Subject: [PATCH 0761/1165] Standardize format of multiSearchFirstIndexXYZ functions --- .../functions/string-search-functions.md | 125 +++++++++++++++++- 1 file changed, 123 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index c4b5fb7c1f5..c067f9010fa 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -294,12 +294,133 @@ multiSearchFirstPosition(haystack, \[needle1, needle2, … Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. -Functions `multiSearchFirstIndexCaseInsensitive`, `multiSearchFirstIndexUTF8` and `multiSearchFirstIndexCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. +Functions [`multiSearchFirstIndexCaseInsensitive`](#multiSearchFirstIndexCaseInsensitive), [`multiSearchFirstIndexUTF8`](#multiSearchFirstIndexUTF8) and [`multiSearchFirstIndexCaseInsensitiveUTF8`](#multiSearchFirstIndexCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** ```sql -multiSearchFirstIndex(haystack, \[needle1, needle2, …, needlen\]) +multiSearchFirstIndex(haystack, [needle1, needle2, ..., needleN]) +``` +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- index (starting from 1) of the leftmost found needle. +- 0, if there was no match. + +**Example** + +Query: + +```sql +SELECT multiSearchFirstIndex('Hello World',['World','Hello']); +``` + +```response +1 +``` + +## multiSearchFirstIndexCaseInsensitive {#multiSearchFirstIndexCaseInsensitive} + +Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. Ignores case. + +**Syntax** + +```sql +multiSearchFirstIndexCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- index (starting from 1) of the leftmost found needle. +- 0, if there was no match. + +**Example** + +Query: + +```sql +SELECT multiSearchFirstIndexCaseInsensitive('hElLo WoRlD',['World','Hello']); +``` + +```response +1 +``` + +## multiSearchFirstIndexUTF8 {#multiSearchFirstIndexUTF8} + +Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. Assumes `haystack` and `needle` are UTF-8 encoded strings. + +**Syntax** + +```sql +multiSearchFirstIndexUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- index (starting from 1) of the leftmost found needle. +- 0, if there was no match. + +**Example** + +Given `Hello World` as a UTF-8 string, find the first index of UTF-8 strings `Hello` and `World`. + +Query: + +```sql +SELECT multiSearchFirstIndexUTF8('\x48\x65\x6c\x6c\x6f\x20\x57\x6f\x72\x6c\x64',['\x57\x6f\x72\x6c\x64','\x48\x65\x6c\x6c\x6f']); +``` + +```response +1 +``` + +## multiSearchFirstIndexCaseInsensitiveUTF8 {#multiSearchFirstIndexCaseInsensitiveUTF8} + +Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. Assumes `haystack` and `needle` are UTF-8 encoded strings. Ignores case. + +**Syntax** + +```sql +multiSearchFirstIndexCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- index (starting from 1) of the leftmost found needle. +- 0, if there was no match. + +**Example** + +Given `HELLO WORLD` as a UTF-8 string, find the first index of UTF-8 strings `hello` and `world`. + +Query: + +```sql +SELECT multiSearchFirstIndexCaseInsensitiveUTF8('\x48\x45\x4c\x4c\x4f\x20\x57\x4f\x52\x4c\x44',['\x68\x65\x6c\x6c\x6f','\x77\x6f\x72\x6c\x64']); +``` + +```response +1 ``` ## multiSearchAny {#multisearchany} From a97a3196b45fa5cb86f7d95b4869318864a18562 Mon Sep 17 00:00:00 2001 From: Blargian Date: Thu, 28 Mar 2024 20:52:55 +0100 Subject: [PATCH 0762/1165] Standardize formatting of multisearchFirstPositionXYZ functions --- .../functions/string-search-functions.md | 128 +++++++++++++++++- 1 file changed, 125 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index c067f9010fa..232397ce592 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -278,16 +278,138 @@ SELECT multiSearchAllPositionsCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\ ["1","6"] ``` -## multiSearchFirstPosition +## multiSearchFirstPosition {#multiSearchFirstPosition} Like `position` but returns the leftmost offset in a `haystack` string which matches any of multiple `needle` strings. -Functions `multiSearchFirstPositionCaseInsensitive`, `multiSearchFirstPositionUTF8` and `multiSearchFirstPositionCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. +Functions [`multiSearchFirstPositionCaseInsensitive`](#multiSearchFirstPositionCaseInsensitive), [`multiSearchFirstPositionUTF8`](#multiSearchFirstPositionUTF8) and [`multiSearchFirstPositionCaseInsensitiveUTF8`](#multiSearchFirstPositionCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** ```sql -multiSearchFirstPosition(haystack, \[needle1, needle2, …, needlen\]) +multiSearchFirstPosition(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings. +- 0, if there was no match. + +**Example** + +Query: + +```sql +SELECT multiSearchFirstPosition('Hello World',['llo', 'Wor', 'ld']); +``` + +```response +3 +``` + +## multiSearchFirstPositionCaseInsensitive + +Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but ignores case. + +**Syntax** + +```sql +multiSearchFirstPositionCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings. +- 0, if there was no match. + +**Example** + +Query: + +```sql +SELECT multiSearchFirstPositionCaseInsensitive('HELLO WORLD',['wor', 'ld', 'ello']); +``` + +```response +2 +``` + +## multiSearchFirstPositionUTF8 {#multiSearchFirstPositionUTF8} + +Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings. + +**Syntax** + +```sql +multiSearchFirstPositionUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings. +- 0, if there was no match. + +**Example** + +Find the leftmost offset in UTF-8 string `hello world` which matches any of the given needles. + +Query: + +```sql +SELECT multiSearchFirstPositionUTF8('\x68\x65\x6c\x6c\x6f\x20\x77\x6f\x72\x6c\x64',['wor', 'ld', 'ello']); +``` + +```response +2 +``` + +## multiSearchFirstPositionCaseInsensitiveUTF8 {#multiSearchFirstPositionCaseInsensitiveUTF8} + +Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings and ignores case. + +**Syntax** + +```sql +multiSearchFirstPositionCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings, ignoring case. +- 0, if there was no match. + +**Example** + +Find the leftmost offset in UTF-8 string `HELLO WORLD` which matches any of the given needles. + +Query: + +```sql +SELECT multiSearchFirstPositionCaseInsensitiveUTF8('\x48\x45\x4c\x4c\x4f\x20\x57\x4f\x52\x4c\x44',['wor', 'ld', 'ello']); +``` + +```response +2 ``` ## multiSearchFirstIndex From 1fde8942d6a4d13a39ac31952bf096054ab7cacc Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 25 Mar 2024 16:45:43 +0100 Subject: [PATCH 0763/1165] Read system.replicas with different max_block_size --- .../02908_many_requests_to_system_replicas.reference | 4 ++++ .../0_stateless/02908_many_requests_to_system_replicas.sh | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference index 17c94686470..0a4008cc35e 100644 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference @@ -1,4 +1,8 @@ Creating 300 tables +900 2700 2700 +900 2700 2700 +900 2700 2700 +900 2700 2700 Making 200 requests to system.replicas Query system.replicas while waiting for other concurrent requests to finish 0 diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 17e1d87963a..18c55159281 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -45,6 +45,13 @@ done wait; +# Check results with different max_block_size +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas), sum(active_replicas) FROM system.replicas WHERE database=currentDatabase()' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas), sum(active_replicas) FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=1' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas), sum(active_replicas) FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=77' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas), sum(active_replicas) FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=11111' + + echo "Making $CONCURRENCY requests to system.replicas" for i in $(seq 1 $CONCURRENCY) From 89afca881f81142481b31682882fdb2e1482f08c Mon Sep 17 00:00:00 2001 From: Blargian Date: Thu, 28 Mar 2024 21:06:17 +0100 Subject: [PATCH 0764/1165] Minor formatting changes to multipleSearchAllPositions --- docs/en/sql-reference/functions/string-search-functions.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 232397ce592..1e564eb4741 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -159,11 +159,12 @@ multiSearchAllPositions(haystack, [needle1, needle2, ..., needleN]) **Arguments** - `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substrings to be searched. Array +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned values** -- Array of the starting position in bytes and counting from 1 (if the substring was found) or 0 (if the substring was not found) +- Array of the starting position in bytes and counting from 1 (if the substring was found). +- 0, if the substring was not found. **Example** From cc35db1cef95a57c458d2606f517b0f43b91c8e4 Mon Sep 17 00:00:00 2001 From: Blargian Date: Thu, 28 Mar 2024 21:20:33 +0100 Subject: [PATCH 0765/1165] Finor fixes --- .../functions/string-search-functions.md | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 1e564eb4741..9430fb0f728 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -192,7 +192,7 @@ multiSearchAllPositionsCaseInsensitive(haystack, [needle1, needle2, ..., needleN **Parameters** - `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -213,7 +213,7 @@ SELECT multiSearchAllPositionsCaseInsensitive('ClickHouse',['c','h']); ## multiSearchAllPositionsUTF8 -Like [multiSearchAllPositions](#multiSearchAllPositions) but assumes `haystack` and the `needle`-s are UTF-8 encoded strings. +Like [multiSearchAllPositions](#multiSearchAllPositions) but assumes `haystack` and the `needle` substrings are UTF-8 encoded strings. **Syntax** @@ -224,7 +224,7 @@ multiSearchAllPositionsUTF8(haystack, [needle1, needle2, ..., needleN]) **Parameters** - `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -233,7 +233,7 @@ multiSearchAllPositionsUTF8(haystack, [needle1, needle2, ..., needleN]) **Example** -Given `ClickHouse` as a UTF-8 string, find the positions of `C` ('\x43') and `H` ('\x48'). +Given `ClickHouse` as a UTF-8 string, find the positions of `C` (`\x43`) and `H` (`\x48`). Query: @@ -258,7 +258,7 @@ multiSearchAllPositionsCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., nee **Parameters** - `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -279,9 +279,9 @@ SELECT multiSearchAllPositionsCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\ ["1","6"] ``` -## multiSearchFirstPosition {#multiSearchFirstPosition} +## multiSearchFirstPosition -Like `position` but returns the leftmost offset in a `haystack` string which matches any of multiple `needle` strings. +Like [`position`](#position) but returns the leftmost offset in a `haystack` string which matches any of multiple `needle` strings. Functions [`multiSearchFirstPositionCaseInsensitive`](#multiSearchFirstPositionCaseInsensitive), [`multiSearchFirstPositionUTF8`](#multiSearchFirstPositionUTF8) and [`multiSearchFirstPositionCaseInsensitiveUTF8`](#multiSearchFirstPositionCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function. @@ -294,7 +294,7 @@ multiSearchFirstPosition(haystack, [needle1, needle2, ..., needleN]) **Parameters** - `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -326,7 +326,7 @@ multiSearchFirstPositionCaseInsensitive(haystack, [needle1, needle2, ..., needle **Parameters** - `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Array of substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — Array of substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -345,7 +345,7 @@ SELECT multiSearchFirstPositionCaseInsensitive('HELLO WORLD',['wor', 'ld', 'ello 2 ``` -## multiSearchFirstPositionUTF8 {#multiSearchFirstPositionUTF8} +## multiSearchFirstPositionUTF8 Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings. @@ -358,7 +358,7 @@ multiSearchFirstPositionUTF8(haystack, [needle1, needle2, ..., needleN]) **Parameters** - `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -379,7 +379,7 @@ SELECT multiSearchFirstPositionUTF8('\x68\x65\x6c\x6c\x6f\x20\x77\x6f\x72\x6c\x6 2 ``` -## multiSearchFirstPositionCaseInsensitiveUTF8 {#multiSearchFirstPositionCaseInsensitiveUTF8} +## multiSearchFirstPositionCaseInsensitiveUTF8 Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings and ignores case. @@ -427,7 +427,7 @@ multiSearchFirstIndex(haystack, [needle1, needle2, ..., needleN]) **Parameters** - `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -446,7 +446,7 @@ SELECT multiSearchFirstIndex('Hello World',['World','Hello']); 1 ``` -## multiSearchFirstIndexCaseInsensitive {#multiSearchFirstIndexCaseInsensitive} +## multiSearchFirstIndexCaseInsensitive Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. Ignores case. @@ -459,7 +459,7 @@ multiSearchFirstIndexCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) **Parameters** - `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -478,7 +478,7 @@ SELECT multiSearchFirstIndexCaseInsensitive('hElLo WoRlD',['World','Hello']); 1 ``` -## multiSearchFirstIndexUTF8 {#multiSearchFirstIndexUTF8} +## multiSearchFirstIndexUTF8 Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. Assumes `haystack` and `needle` are UTF-8 encoded strings. @@ -512,7 +512,7 @@ SELECT multiSearchFirstIndexUTF8('\x48\x65\x6c\x6c\x6f\x20\x57\x6f\x72\x6c\x64', 1 ``` -## multiSearchFirstIndexCaseInsensitiveUTF8 {#multiSearchFirstIndexCaseInsensitiveUTF8} +## multiSearchFirstIndexCaseInsensitiveUTF8 Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. Assumes `haystack` and `needle` are UTF-8 encoded strings. Ignores case. @@ -525,7 +525,7 @@ multiSearchFirstIndexCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needl **Parameters** - `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -546,7 +546,7 @@ SELECT multiSearchFirstIndexCaseInsensitiveUTF8('\x48\x45\x4c\x4c\x4f\x20\x57\x4 1 ``` -## multiSearchAny {#multisearchany} +## multiSearchAny Returns 1, if at least one string needlei matches the string `haystack` and 0 otherwise. @@ -561,7 +561,7 @@ multiSearchAny(haystack, [needle1, needle2, ..., needleN]) **Parameters** - `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -580,7 +580,7 @@ SELECT multiSearchAny('ClickHouse',['C','H']); 1 ``` -## multiSearchAnyCaseInsensitive {#multiSearchAnyCaseInsensitive} +## multiSearchAnyCaseInsensitive Like [multiSearchAny](#multisearchany) but ignores case. @@ -612,9 +612,9 @@ SELECT multiSearchAnyCaseInsensitive('ClickHouse',['c','h']); 1 ``` -## multiSearchAnyUTF8 {#multiSearchAnyUTF8} +## multiSearchAnyUTF8 -Like [multiSearchAny](#multisearchany) but assumes `haystack` and the `needle`-s are UTF-8 encoded strings. +Like [multiSearchAny](#multisearchany) but assumes `haystack` and the `needle` substrings are UTF-8 encoded strings. *Syntax** @@ -625,7 +625,7 @@ multiSearchAnyUTF8(haystack, [needle1, needle2, ..., needleN]) **Parameters** - `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) +- `needle` — UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -646,7 +646,7 @@ SELECT multiSearchAnyUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x43','\x 1 ``` -## multiSearchAnyCaseInsensitiveUTF8 {#multiSearchAnyCaseInsensitiveUTF8} +## multiSearchAnyCaseInsensitiveUTF8 Like [multiSearchAnyUTF8](#multiSearchAnyUTF8) but ignores case. From 7288aeb826ec4ea964bb6e408c7644a2f53cf7c1 Mon Sep 17 00:00:00 2001 From: Blargian Date: Thu, 28 Mar 2024 21:45:36 +0100 Subject: [PATCH 0766/1165] More consistency edits --- .../functions/string-search-functions.md | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 9430fb0f728..7ffaee53f89 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -126,6 +126,8 @@ Like [position](#position) but assumes `haystack` and `needle` are UTF-8 encoded Function `positionUTF8` correctly counts character `ö` (represented by two points) as a single Unicode codepoint: +Query: + ``` sql SELECT positionUTF8('Motörhead', 'r'); ``` @@ -163,11 +165,13 @@ multiSearchAllPositions(haystack, [needle1, needle2, ..., needleN]) **Returned values** -- Array of the starting position in bytes and counting from 1 (if the substring was found). +- Array of the starting position in bytes and counting from 1, if the substring was found. - 0, if the substring was not found. **Example** +Query: + ``` sql SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']); ``` @@ -207,6 +211,8 @@ Query: SELECT multiSearchAllPositionsCaseInsensitive('ClickHouse',['c','h']); ``` +Result: + ```response ["1","6"] ``` @@ -241,6 +247,8 @@ Query: SELECT multiSearchAllPositionsUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x43','\x48']); ``` +Result: + ```response ["1","6"] ``` @@ -275,6 +283,8 @@ Query: SELECT multiSearchAllPositionsCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x63','\x68']); ``` +Result: + ```response ["1","6"] ``` @@ -309,6 +319,8 @@ Query: SELECT multiSearchFirstPosition('Hello World',['llo', 'Wor', 'ld']); ``` +Result: + ```response 3 ``` @@ -341,6 +353,8 @@ Query: SELECT multiSearchFirstPositionCaseInsensitive('HELLO WORLD',['wor', 'ld', 'ello']); ``` +Result: + ```response 2 ``` @@ -375,6 +389,8 @@ Query: SELECT multiSearchFirstPositionUTF8('\x68\x65\x6c\x6c\x6f\x20\x77\x6f\x72\x6c\x64',['wor', 'ld', 'ello']); ``` +Result: + ```response 2 ``` @@ -409,6 +425,8 @@ Query: SELECT multiSearchFirstPositionCaseInsensitiveUTF8('\x48\x45\x4c\x4c\x4f\x20\x57\x4f\x52\x4c\x44',['wor', 'ld', 'ello']); ``` +Result: + ```response 2 ``` @@ -442,6 +460,8 @@ Query: SELECT multiSearchFirstIndex('Hello World',['World','Hello']); ``` +Result: + ```response 1 ``` @@ -474,6 +494,8 @@ Query: SELECT multiSearchFirstIndexCaseInsensitive('hElLo WoRlD',['World','Hello']); ``` +Result: + ```response 1 ``` @@ -508,6 +530,8 @@ Query: SELECT multiSearchFirstIndexUTF8('\x48\x65\x6c\x6c\x6f\x20\x57\x6f\x72\x6c\x64',['\x57\x6f\x72\x6c\x64','\x48\x65\x6c\x6c\x6f']); ``` +Result: + ```response 1 ``` @@ -542,6 +566,8 @@ Query: SELECT multiSearchFirstIndexCaseInsensitiveUTF8('\x48\x45\x4c\x4c\x4f\x20\x57\x4f\x52\x4c\x44',['\x68\x65\x6c\x6c\x6f','\x77\x6f\x72\x6c\x64']); ``` +Result: + ```response 1 ``` @@ -576,6 +602,8 @@ Query: SELECT multiSearchAny('ClickHouse',['C','H']); ``` +Result: + ```response 1 ``` @@ -608,6 +636,8 @@ Query: SELECT multiSearchAnyCaseInsensitive('ClickHouse',['c','h']); ``` +Result: + ```response 1 ``` @@ -642,6 +672,8 @@ Query: SELECT multiSearchAnyUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x43','\x48']); ``` +Result: + ```response 1 ``` @@ -676,6 +708,8 @@ Query: SELECT multiSearchAnyCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x68']); ``` +Result: + ```response 1 ``` From 1dcba74f66d64bf7d77b7da585cabc16a5d4e29f Mon Sep 17 00:00:00 2001 From: Blargian Date: Thu, 28 Mar 2024 21:54:26 +0100 Subject: [PATCH 0767/1165] Small grammar edits to description at top of the page --- .../functions/string-search-functions.md | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 7ffaee53f89..8aff8b7e930 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -6,14 +6,17 @@ sidebar_label: Searching in Strings # Functions for Searching in Strings -All functions in this section search by default case-sensitively. Case-insensitive search is usually provided by separate function variants. -Note that case-insensitive search follows the lowercase-uppercase rules of the English language. E.g. Uppercased `i` in English language is -`I` whereas in Turkish language it is `İ` - results for languages other than English may be unexpected. +All functions in this section search case-sensitively by default. Case-insensitive search is usually provided by separate function variants. -Functions in this section also assume that the searched string and the search string are single-byte encoded text. If this assumption is +:::note +Case-insensitive search follows the lowercase-uppercase rules of the English language. E.g. Uppercased `i` in the English language is +`I` whereas in the Turkish language it is `İ` - results for languages other than English may be unexpected. +::: + +Functions in this section also assume that the searched string (refered to in this section as `haystack`) and the search string (refered to in this section as `needle`) are single-byte encoded text. If this assumption is violated, no exception is thrown and results are undefined. Search with UTF-8 encoded strings is usually provided by separate function variants. Likewise, if a UTF-8 function variant is used and the input strings are not UTF-8 encoded text, no exception is thrown and the -results are undefined. Note that no automatic Unicode normalization is performed, you can use the +results are undefined. Note that no automatic Unicode normalization is performed, however you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that. [General strings functions](string-functions.md) and [functions for replacing in strings](string-replace-functions.md) are described separately. @@ -55,6 +58,8 @@ Type: `Integer`. **Examples** +Query: + ``` sql SELECT position('Hello, world!', '!'); ``` @@ -69,12 +74,16 @@ Result: Example with `start_pos` argument: +Query: + ``` sql SELECT position('Hello, world!', 'o', 1), position('Hello, world!', 'o', 7) ``` +Result: + ``` text ┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐ │ 5 │ 9 │ @@ -83,6 +92,8 @@ SELECT Example for `needle IN haystack` syntax: +Query: + ```sql SELECT 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); ``` @@ -97,6 +108,8 @@ Result: Examples with empty `needle` substring: +Query: + ``` sql SELECT position('abc', ''), @@ -108,6 +121,8 @@ SELECT position('abc', '', 5) ``` +Result: + ``` text ┌─position('abc', '')─┬─position('abc', '', 0)─┬─position('abc', '', 1)─┬─position('abc', '', 2)─┬─position('abc', '', 3)─┬─position('abc', '', 4)─┬─position('abc', '', 5)─┐ │ 1 │ 1 │ 1 │ 2 │ 3 │ 4 │ 0 │ @@ -116,7 +131,23 @@ SELECT ## positionCaseInsensitive -Like [position](#position) but searches case-insensitively. +A case insensitive invariant of [position](#position). + +**Example** + +Query: + +``` sql +SELECT position('Hello, world!', 'hello'); +``` + +Result: + +``` text +┌─position('Hello, world!', 'hello')─┐ +│ 0 │ +└────────────────────────────────────┘ +``` ## positionUTF8 From 79b486b8bfa9d9e540d1ab95281e4d4ca554f492 Mon Sep 17 00:00:00 2001 From: zhongyuankai <872237106@qq.com> Date: Fri, 29 Mar 2024 09:49:24 +0800 Subject: [PATCH 0768/1165] batter --- src/Interpreters/InterpreterDropQuery.cpp | 2 +- src/Parsers/ASTIdentifier.cpp | 6 ------ src/Parsers/ASTIdentifier.h | 1 - src/Parsers/tests/gtest_dictionary_parser.cpp | 4 ++-- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 7c5d0de081a..e29e59ee4c3 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -465,7 +465,7 @@ void InterpreterDropQuery::extendQueryLogElemImpl(DB::QueryLogElement & elem, co throw Exception(ErrorCodes::SYNTAX_ERROR, "Unexpected type for list of table names."); String query_database = identifier->getDatabaseName(); - String query_table = identifier->getTableName(); + String query_table = identifier->shortName(); if (!query_database.empty() && query_table.empty()) { elem.query_databases.insert(backQuoteIfNeed(query_database)); diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 1a24dac61e6..80a618170c6 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -207,12 +207,6 @@ String ASTTableIdentifier::getDatabaseName() const else return {}; } -String ASTTableIdentifier::getTableName() const -{ - if (name_parts.size() == 2) return name_parts[1]; - else return name_parts[0]; -} - ASTPtr ASTTableIdentifier::getTable() const { if (name_parts.size() == 2) diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index d75b5a99dbe..d986b9170f3 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -86,7 +86,6 @@ public: StorageID getTableId() const; String getDatabaseName() const; - String getTableName() const; ASTPtr getTable() const; ASTPtr getDatabase() const; diff --git a/src/Parsers/tests/gtest_dictionary_parser.cpp b/src/Parsers/tests/gtest_dictionary_parser.cpp index f9074c0b2eb..44205975cdc 100644 --- a/src/Parsers/tests/gtest_dictionary_parser.cpp +++ b/src/Parsers/tests/gtest_dictionary_parser.cpp @@ -304,7 +304,7 @@ TEST(ParserDictionaryDDL, ParseDropQuery) auto & database_and_tables1 = drop1->database_and_tables->as(); auto identifier1 = dynamic_pointer_cast(database_and_tables1.children[0]); EXPECT_EQ(identifier1->getDatabaseName(), "test"); - EXPECT_EQ(identifier1->getTableName(), "dict1"); + EXPECT_EQ(identifier1->shortName(), "dict1"); auto str1 = serializeAST(*drop1); EXPECT_EQ(input1, str1); @@ -317,7 +317,7 @@ TEST(ParserDictionaryDDL, ParseDropQuery) auto & database_and_tables2 = drop2->database_and_tables->as(); auto identifier2 = dynamic_pointer_cast(database_and_tables2.children[0]); EXPECT_EQ(identifier2->getDatabaseName(), ""); - EXPECT_EQ(identifier2->getTableName(), "dict2"); + EXPECT_EQ(identifier2->shortName(), "dict2"); auto str2 = serializeAST(*drop2); EXPECT_EQ(input2, str2); } From ba62b01f4d15cc798e19a67a77dcc29daf7a7b8d Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Fri, 29 Mar 2024 02:23:53 +0000 Subject: [PATCH 0769/1165] remove dead code --- src/Storages/MergeTree/MergeTreeSelectProcessor.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 01bb3851e04..6b663e0fd36 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -66,15 +66,6 @@ public: void addPartLevelToChunk(bool add_part_level_) { add_part_level = add_part_level_; } private: - /// This struct allow to return block with no columns but with non-zero number of rows similar to Chunk - struct BlockAndProgress - { - Block block; - size_t row_count = 0; - size_t num_read_rows = 0; - size_t num_read_bytes = 0; - }; - /// Sets up range readers corresponding to data readers void initializeRangeReaders(); From c5f88f6eddd7c0222dd3427ac75ebd1ca125718d Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Fri, 29 Mar 2024 15:40:14 +0800 Subject: [PATCH 0770/1165] Improve hive text read by remove setting --- src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp | 10 +--------- src/Processors/Formats/Impl/HiveTextRowInputFormat.h | 2 -- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp index 9f2a795427c..225ee8341f6 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp @@ -15,10 +15,10 @@ static FormatSettings updateFormatSettings(const FormatSettings & settings, cons { FormatSettings updated = settings; updated.skip_unknown_fields = true; - updated.with_names_use_header = true; updated.date_time_input_format = FormatSettings::DateTimeInputFormat::BestEffort; updated.defaults_for_omitted_fields = true; updated.csv.delimiter = updated.hive_text.fields_delimiter; + updated.csv.allow_variable_number_of_columns = true; if (settings.hive_text.input_field_names.empty()) updated.hive_text.input_field_names = header.getNames(); return updated; @@ -42,14 +42,6 @@ HiveTextFormatReader::HiveTextFormatReader(PeekableReadBuffer & buf_, const Form { } -std::vector HiveTextFormatReader::readNames() -{ - PeekableReadBufferCheckpoint checkpoint{*buf, true}; - auto values = readHeaderRow(); - input_field_names.resize(values.size()); - return input_field_names; -} - std::vector HiveTextFormatReader::readTypes() { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "HiveTextRowInputFormat::readTypes is not implemented"); diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.h b/src/Processors/Formats/Impl/HiveTextRowInputFormat.h index 313aad0d40d..71a8e607807 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.h +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.h @@ -27,8 +27,6 @@ class HiveTextFormatReader final : public CSVFormatReader { public: HiveTextFormatReader(PeekableReadBuffer & buf_, const FormatSettings & format_settings_); - - std::vector readNames() override; std::vector readTypes() override; private: From 14882d9a7d7a7da867d1b8626f756e8e0d7e2a21 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Fri, 29 Mar 2024 16:06:24 +0800 Subject: [PATCH 0771/1165] remove readNames code --- src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp | 5 +++++ src/Processors/Formats/Impl/HiveTextRowInputFormat.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp index 225ee8341f6..f8652a430df 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp @@ -42,6 +42,11 @@ HiveTextFormatReader::HiveTextFormatReader(PeekableReadBuffer & buf_, const Form { } +std::vector HiveTextFormatReader::readNames() +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "HiveTextRowInputFormat::readNames is not implemented"); +} + std::vector HiveTextFormatReader::readTypes() { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "HiveTextRowInputFormat::readTypes is not implemented"); diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.h b/src/Processors/Formats/Impl/HiveTextRowInputFormat.h index 71a8e607807..313aad0d40d 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.h +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.h @@ -27,6 +27,8 @@ class HiveTextFormatReader final : public CSVFormatReader { public: HiveTextFormatReader(PeekableReadBuffer & buf_, const FormatSettings & format_settings_); + + std::vector readNames() override; std::vector readTypes() override; private: From 9da4ce29dbfa694190ae05e27e502784c046282c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 Mar 2024 16:16:15 +0100 Subject: [PATCH 0772/1165] Revert output Pretty in tty --- src/Client/ClientBase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 767a9b2b9f9..7a3192d1d9c 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -753,7 +753,7 @@ void ClientBase::setDefaultFormatsFromConfiguration() else default_output_format = "TSV"; } - else if (is_interactive || stdout_is_a_tty) + else if (is_interactive) { default_output_format = "PrettyCompact"; } From 3c07d411521284fbc5535d6d36780b1a8a227406 Mon Sep 17 00:00:00 2001 From: Blargian Date: Fri, 29 Mar 2024 17:24:54 +0100 Subject: [PATCH 0773/1165] Add minimal documentation for hasThreadFuzzer --- docs/en/sql-reference/functions/other-functions.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 2638a5a650b..9307729beed 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -591,6 +591,16 @@ SELECT hasColumnInTable('system','metrics','non-existing_column') 0 ``` +## hasThreadFuzzer + +Returns whether Thread Fuzzer is effective. It can be used in tests to prevent too long runs. + +**Syntax** + +```sql +hasThreadFuzzer(); +``` + ## bar Builds a bar chart. From c946a3a89c623797895fc2abe0cc0a22fc7a6114 Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Fri, 29 Mar 2024 17:59:47 +0100 Subject: [PATCH 0774/1165] Update storing-data.md --- docs/en/operations/storing-data.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index fd81bc197d1..9ffbb64c1ed 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -520,13 +520,13 @@ Example of configuration for versions later or equal to 22.8: - +
cache
- + ``` @@ -546,13 +546,13 @@ Example of configuration for versions earlier than 22.8: - +
s3
- + ``` From b0865cf2e37b3dd5cd636444523d037140e12887 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 30 Mar 2024 00:14:03 +0100 Subject: [PATCH 0775/1165] Fix restoring parts while a storage is readonly. --- .../MergeTree/ReplicatedMergeTreeSink.cpp | 15 ++++++++------- src/Storages/MergeTree/ReplicatedMergeTreeSink.h | 7 ++++--- src/Storages/StorageReplicatedMergeTree.cpp | 8 ++++++-- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index b43d47bf5f4..fd5f3853a6b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -128,7 +128,8 @@ ReplicatedMergeTreeSinkImpl::ReplicatedMergeTreeSinkImpl( bool deduplicate_, bool majority_quorum, ContextPtr context_, - bool is_attach_) + bool is_attach_, + bool allow_attach_while_readonly_) : SinkToStorage(metadata_snapshot_->getSampleBlock()) , storage(storage_) , metadata_snapshot(metadata_snapshot_) @@ -136,6 +137,7 @@ ReplicatedMergeTreeSinkImpl::ReplicatedMergeTreeSinkImpl( , quorum_timeout_ms(quorum_timeout_ms_) , max_parts_per_block(max_parts_per_block_) , is_attach(is_attach_) + , allow_attach_while_readonly(allow_attach_while_readonly_) , quorum_parallel(quorum_parallel_) , deduplicate(deduplicate_) , log(getLogger(storage.getLogName() + " (Replicated OutputStream)")) @@ -440,7 +442,7 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithF try { - bool deduplicated = commitPart(zookeeper, part, partition.block_id, delayed_chunk->replicas_num, false).second; + bool deduplicated = commitPart(zookeeper, part, partition.block_id, delayed_chunk->replicas_num).second; last_block_is_duplicate = last_block_is_duplicate || deduplicated; @@ -485,7 +487,7 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithFa while (true) { partition.temp_part.finalize(); - auto conflict_block_ids = commitPart(zookeeper, partition.temp_part.part, partition.block_id, delayed_chunk->replicas_num, false).first; + auto conflict_block_ids = commitPart(zookeeper, partition.temp_part.part, partition.block_id, delayed_chunk->replicas_num).first; if (conflict_block_ids.empty()) { auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); @@ -556,7 +558,7 @@ bool ReplicatedMergeTreeSinkImpl::writeExistingPart(MergeTreeData::Mutabl { part->version.setCreationTID(Tx::PrehistoricTID, nullptr); String block_id = deduplicate ? fmt::format("{}_{}", part->info.partition_id, part->checksums.getTotalChecksumHex()) : ""; - bool deduplicated = commitPart(zookeeper, part, block_id, replicas_num, /* writing_existing_part */ true).second; + bool deduplicated = commitPart(zookeeper, part, block_id, replicas_num).second; /// Set a special error code if the block is duplicate int error = (deduplicate && deduplicated) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; @@ -647,8 +649,7 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: const ZooKeeperWithFaultInjectionPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const BlockIDsType & block_id, - size_t replicas_num, - bool writing_existing_part) + size_t replicas_num) { /// It is possible that we alter a part with different types of source columns. /// In this case, if column was not altered, the result type will be different with what we have in metadata. @@ -800,7 +801,7 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: /// When we attach existing parts it's okay to be in read-only mode /// For example during RESTORE REPLICA. - if (!writing_existing_part) + if (!allow_attach_while_readonly) { retries_ctl.setUserError( Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode: replica_path={}", storage.replica_path)); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index 29f3183be64..39623c20584 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -45,7 +45,8 @@ public: ContextPtr context_, // special flag to determine the ALTER TABLE ATTACH PART without the query context, // needed to set the special LogEntryType::ATTACH_PART - bool is_attach_ = false); + bool is_attach_ = false, + bool allow_attach_while_readonly_ = false); ~ReplicatedMergeTreeSinkImpl() override; @@ -93,8 +94,7 @@ private: const ZooKeeperWithFaultInjectionPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const BlockIDsType & block_id, - size_t replicas_num, - bool writing_existing_part); + size_t replicas_num); /// Wait for quorum to be satisfied on path (quorum_path) form part (part_name) @@ -123,6 +123,7 @@ private: UInt64 cache_version = 0; bool is_attach = false; + bool allow_attach_while_readonly = false; bool quorum_parallel = false; const bool deduplicate = true; bool last_block_is_duplicate = false; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 951a0facc6b..7ca508c362d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -6512,7 +6512,7 @@ PartitionCommandsResultInfo StorageReplicatedMergeTree::attachPartition( /// TODO Allow to use quorum here. ReplicatedMergeTreeSink output(*this, metadata_snapshot, /* quorum */ 0, /* quorum_timeout_ms */ 0, /* max_parts_per_block */ 0, /* quorum_parallel */ false, query_context->getSettingsRef().insert_deduplicate, - /* majority_quorum */ false, query_context, /*is_attach*/true); + /* majority_quorum */ false, query_context, /* is_attach */ true, /* allow_attach_while_readonly */ true); for (size_t i = 0; i < loaded_parts.size(); ++i) { @@ -10500,7 +10500,11 @@ void StorageReplicatedMergeTree::restoreDataFromBackup(RestorerFromBackup & rest void StorageReplicatedMergeTree::attachRestoredParts(MutableDataPartsVector && parts) { auto metadata_snapshot = getInMemoryMetadataPtr(); - auto sink = std::make_shared(*this, metadata_snapshot, 0, 0, 0, false, false, false, getContext(), /*is_attach*/true); + + auto sink = std::make_shared( + *this, metadata_snapshot, /* quorum */ 0, /* quorum_timeout_ms */ 0, /* max_parts_per_block */ 0, /* quorum_parallel */ false, + /* deduplicate */ false, /* majority_quorum */ false, getContext(), /* is_attach */ true, /* allow_attach_while_readonly */ false); + for (auto part : parts) sink->writeExistingPart(part); } From 9182789ed8bd784f0987d60219626afffdb72a2f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 30 Mar 2024 02:36:03 +0100 Subject: [PATCH 0776/1165] Update test --- tests/queries/0_stateless/03021_output_format_tty.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/03021_output_format_tty.sh b/tests/queries/0_stateless/03021_output_format_tty.sh index dfc37c3b30a..55dedeb6773 100755 --- a/tests/queries/0_stateless/03021_output_format_tty.sh +++ b/tests/queries/0_stateless/03021_output_format_tty.sh @@ -21,12 +21,6 @@ expect_after { -i $any_spawn_id timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT --query 'SELECT 1'" -expect "│ 1 │" -expect "└───┘" -expect eof - -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --query 'SELECT 2'" -expect "│ 2 │" -expect "└───┘" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT --query 'SELECT 1, 2'" +expect "1\t2" expect eof From 49c33039b9f53a2f280b8ca74aeea1d119ce9a2d Mon Sep 17 00:00:00 2001 From: Blargian Date: Sat, 30 Mar 2024 14:13:30 +0100 Subject: [PATCH 0777/1165] Updates to hasSubsequence functions --- .../functions/string-search-functions.md | 106 +++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 818626fc4fc..f373353d3f6 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -662,8 +662,10 @@ Type: `UInt8`. **Examples** +Query: + ``` sql -SELECT hasSubsequence('garbage', 'arg') ; +SELECT hasSubsequence('garbage', 'arg'); ``` Result: @@ -678,14 +680,116 @@ Result: Like [hasSubsequence](#hasSubsequence) but searches case-insensitively. +**Syntax** + +``` sql +hasSubsequence(haystack, needle) +``` + +**Arguments** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Subsequence to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). + +**Returned values** + +- 1, if needle is a subsequence of haystack. +- 0, otherwise. + +Type: `UInt8`. + +**Examples** + +Query: + +``` sql +SELECT hasSubsequenceCaseInsensitive('garbage', 'ARG'); +``` + +Result: + +``` text +┌─hasSubsequenceCaseInsensitive('garbage', 'ARG')─┐ +│ 1 │ +└─────────────────────────────────────────────────┘ +``` + ## hasSubsequenceUTF8 Like [hasSubsequence](#hasSubsequence) but assumes `haystack` and `needle` are UTF-8 encoded strings. +**Syntax** + +``` sql +hasSubsequenceUTF8(haystack, needle) +``` + +**Arguments** + +- `haystack` — String in which the search is performed. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Subsequence to be searched. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). + +**Returned values** + +- 1, if needle is a subsequence of haystack. +- 0, otherwise. + +Type: `UInt8`. + +Query: + +**Examples** + +``` sql +select hasSubsequenceUTF8('ClickHouse - столбцовая система управления базами данных', 'система'); +``` + +Result: + +``` text +┌─hasSubsequenceUTF8('ClickHouse - столбцовая система управления базами данных', 'система')─┐ +│ 1 │ +└───────────────────────────────────────────────────────────────────────────────────────────┘ +``` + ## hasSubsequenceCaseInsensitiveUTF8 Like [hasSubsequenceUTF8](#hasSubsequenceUTF8) but searches case-insensitively. +**Syntax** + +``` sql +hasSubsequenceCaseInsensitiveUTF8(haystack, needle) +``` + +**Arguments** + +- `haystack` — String in which the search is performed. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Subsequence to be searched. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). + +**Returned values** + +- 1, if needle is a subsequence of haystack. +- 0, otherwise. + +Type: `UInt8`. + +**Examples** + +Query: + +``` sql +select hasSubsequenceCaseInsensitiveUTF8('ClickHouse - столбцовая система управления базами данных', 'СИСТЕМА'); +``` + +Result: + +``` text +┌─hasSubsequenceCaseInsensitiveUTF8('ClickHouse - столбцовая система управления базами данных', 'СИСТЕМА')─┐ +│ 1 │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + ## hasToken Returns 1 if a given token is present in a haystack, or 0 otherwise. From 33136b9a1a0ffb10b81cee954a875f8f233867e2 Mon Sep 17 00:00:00 2001 From: Blargian Date: Sat, 30 Mar 2024 15:16:01 +0100 Subject: [PATCH 0778/1165] Minor detail fixes --- docs/en/sql-reference/functions/other-functions.md | 2 +- docs/en/sql-reference/functions/string-search-functions.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 9307729beed..feead3c3b5e 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -593,7 +593,7 @@ SELECT hasColumnInTable('system','metrics','non-existing_column') ## hasThreadFuzzer -Returns whether Thread Fuzzer is effective. It can be used in tests to prevent too long runs. +Returns whether Thread Fuzzer is effective. It can be used in tests to prevent runs from being too long. **Syntax** diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index f373353d3f6..d613a100a27 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -683,7 +683,7 @@ Like [hasSubsequence](#hasSubsequence) but searches case-insensitively. **Syntax** ``` sql -hasSubsequence(haystack, needle) +hasSubsequenceCaseInsensitive(haystack, needle) ``` **Arguments** @@ -908,7 +908,7 @@ Returns 1 if a given token is present in a haystack, 0 otherwise. Ignores case a **Syntax** ```sql -hasTokenCaseInsensitive(haystack, token) +hasTokenCaseInsensitiveOrNull(haystack, token) ``` **Parameters** From 2f50c76f3d40324cd5ea201b2acf6d5272f2bb5e Mon Sep 17 00:00:00 2001 From: Blargian Date: Sat, 30 Mar 2024 16:30:55 +0100 Subject: [PATCH 0779/1165] Update ngramDistance functions --- .../functions/string-search-functions.md | 136 +++++++++++++++++- 1 file changed, 134 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 22f879c62ae..145228ae727 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -465,9 +465,9 @@ Alias: `haystack NOT ILIKE pattern` (operator) ## ngramDistance -Calculates the 4-gram distance between a `haystack` string and a `needle` string. For that, it counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns a Float32 between 0 and 1. The smaller the result is, the more strings are similar to each other. Throws an exception if constant `needle` or `haystack` arguments are more than 32Kb in size. If any of non-constant `haystack` or `needle` arguments is more than 32Kb in size, the distance is always 1. +Calculates the 4-gram distance between a `haystack` string and a `needle` string. For this, it counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns a [Float32](../../sql-reference/data-types/float.md/#float32-float64) between 0 and 1. The smaller the result is, the more similar the strings are to each other. -Functions `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. +Functions [`ngramDistanceCaseInsensitive`](#ngramdistancecaseinsensitive), [`ngramDistanceUTF8`](#ngramdistanceutf8), [`ngramDistanceCaseInsensitiveUTF8`](#ngramdistancecaseinsensitiveutf8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** @@ -475,6 +475,138 @@ Functions `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseIns ngramDistance(haystack, needle) ``` +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Implementation details** + +This function will throw an exception if constant `needle` or `haystack` arguments are more than 32Kb in size. If any non-constant `haystack` or `needle` arguments are more than 32Kb in size, then the distance is always 1. + +**Examples** + +The more similar two strings are to each other, the closer the result will be to 0 (identical). + +Query: + +```sql +SELECT ngramDistance('ClickHouse','ClickHouse!'); +``` + +Result: + +```response +0.06666667 +``` + +The less similar two strings are to each, the larger the result will be. + + +Query: + +```sql +SELECT ngramDistance('ClickHouse','House'); +``` + +Result: + +```response +0.5555556 +``` + +## ngramDistanceCaseInsensitive + +Provides a case-insensitve variant of [ngramDistance](#ngramdistance). + +**Syntax** + +```sql +ngramDistanceCaseInsensitive(haystack, needle) +``` + +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Examples** + +With [ngramDistance](#ngramdistance) differences in case will drive up the similarity score: + +Query: + +```sql +SELECT ngramDistance('ClickHouse','clickhouse'); +``` + +Result: + +```response +0.71428573 +``` + +With [ngramDistanceCaseInsensitive](#ngramdistancecaseinsensitive) case is ignored so two identical strings differing only in case will now read as identical: + +Query: + +```sql +SELECT ngramDistanceCaseInsensitive('ClickHouse','clickhouse'); +``` + +Result: + +```response +0 +``` + +## ngramDistanceUTF8 + +Provides a UTF-8 variant of [ngramDistance](#ngramdistance). Assumes that `needle` and `haystack` strings are UTF-8 encoded strings. + +**Syntax** + +```sql +ngramDistanceUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +## ngramDistanceCaseInsensitiveUTF8 + +Provides a case-insensitive variant of [ngramDistanceUTF8](#ngramdistanceutf8). + +**Syntax** + +```sql +ngramDistanceCaseInsensitiveUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + + ## ngramSearch Like `ngramDistance` but calculates the non-symmetric difference between a `needle` string and a `haystack` string, i.e. the number of n-grams from needle minus the common number of n-grams normalized by the number of `needle` n-grams. Returns a Float32 between 0 and 1. The bigger the result is, the more likely `needle` is in the `haystack`. This function is useful for fuzzy string search. Also see function `soundex`. From 380788ea30ff2aba639afc5f3568bbe599d4d835 Mon Sep 17 00:00:00 2001 From: Blargian Date: Sat, 30 Mar 2024 17:22:31 +0100 Subject: [PATCH 0780/1165] Update ngramSearch functions --- .../functions/string-search-functions.md | 142 +++++++++++++++++- 1 file changed, 136 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 145228ae727..82226caeba3 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -609,13 +609,9 @@ ngramDistanceCaseInsensitiveUTF8(haystack, needle) ## ngramSearch -Like `ngramDistance` but calculates the non-symmetric difference between a `needle` string and a `haystack` string, i.e. the number of n-grams from needle minus the common number of n-grams normalized by the number of `needle` n-grams. Returns a Float32 between 0 and 1. The bigger the result is, the more likely `needle` is in the `haystack`. This function is useful for fuzzy string search. Also see function `soundex`. +Like `ngramDistance` but calculates the non-symmetric difference between a `needle` string and a `haystack` string, i.e. the number of n-grams from the needle minus the common number of n-grams normalized by the number of `needle` n-grams. Returns a [Float32](../../sql-reference/data-types/float.md/#float32-float64) between 0 and 1. The bigger the result is, the more likely `needle` is in the `haystack`. This function is useful for fuzzy string search. Also see function [`soundex`](../../sql-reference/functions/string-functions#soundex). -Functions `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. - -:::note -The UTF-8 variants use the 3-gram distance. These are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables – collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function – we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one – this works for Latin and mostly for all Cyrillic letters. -::: +Functions [`ngramSearchCaseInsensitive`](#ngramsearchcaseinsensitive), [`ngramSearchUTF8`](#ngramsearchutf8), [`ngramSearchCaseInsensitiveUTF8`](#ngramsearchcaseinsensitiveutf8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** @@ -623,6 +619,140 @@ The UTF-8 variants use the 3-gram distance. These are not perfectly fair n-gram ngramSearch(haystack, needle) ``` +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Implementation details** + +:::note +The UTF-8 variants use the 3-gram distance. These are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables – collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function – we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one – this works for Latin and mostly for all Cyrillic letters. +::: + +**Example** + +Query: + +```sql +SELECT ngramSearch('Hello World','World Hello'); +``` + +Result: + +```response +0.5 +``` + +## ngramSearchCaseInsensitive + +Provides a case-insensitive variant of [ngramSearch](#ngramSearch). + +**Syntax** + +```sql +ngramSearchCaseInsensitive(haystack, needle) +``` + +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +The bigger the result is, the more likely `needle` is in the `haystack`. + +**Example** + +Query: + +```sql +SELECT ngramSearchCaseInsensitive('Hello World','hello'); +``` + +Result: + +```response +1 +``` + +## ngramSearchUTF8 + +Provides a UTF-8 variant of [ngramSearch](#ngramsearch) in which `needle` and `haystack` are assumed to be UTF-8 encoded strings. + +**Syntax** + +```sql +ngramSearchUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +The bigger the result is, the more likely `needle` is in the `haystack`. + +**Example** + +Query: + +```sql +SELECT ngramSearchUTF8('абвгдеёжз', 'гдеёзд'); +``` + +Result: + +```response +0.5 +``` + +## ngramSearchCaseInsensitiveUTF8 + +Provides a case-insensitive variant of [ngramSearchUTF8](#ngramsearchutf8) in which `needle` and `haystack`. + +**Syntax** + +```sql +ngramSearchCaseInsensitiveUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +The bigger the result is, the more likely `needle` is in the `haystack`. + +**Example** + +Query: + +```sql +SELECT ngramSearchCaseInsensitiveUTF8('абвГДЕёжз', 'АбвгдЕЁжз'); +``` + +Result: + +```response +0.57142854 +``` + ## countSubstrings Returns how often substring `needle` occurs in string `haystack`. From 6b9b597051636661784cfb1a283a1364104443a7 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Sat, 30 Mar 2024 21:34:01 +0300 Subject: [PATCH 0781/1165] Crash in Engine Merge if Row Policy does not have expression (#61971) --- src/Storages/StorageMerge.cpp | 2 +- .../02763_row_policy_storage_merge.reference | 8 +++++++ .../02763_row_policy_storage_merge.sql.j2 | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index caec03c95b3..1b5e4860464 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -583,7 +583,7 @@ std::vector ReadFromMerge::createChildrenPlans(SelectQ database_name, table_name, RowPolicyFilterType::SELECT_FILTER); - if (row_policy_filter_ptr) + if (row_policy_filter_ptr && !row_policy_filter_ptr->empty()) { row_policy_data_opt = RowPolicyData(row_policy_filter_ptr, storage, modified_context); row_policy_data_opt->extendNames(real_column_names); diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference index 9fa5612e7cd..6510a3c933e 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -166,6 +166,10 @@ SELECT x, y from merge(currentDatabase(), 02763_merge 4 14 4 14 4 14 +USING 0 +USING 1 +zzz +==== SETTINGS optimize_move_to_prewhere= 1 SELECT * FROM 02763_merge_log_1 3 13 @@ -312,3 +316,7 @@ SELECT x, y from merge(currentDatabase(), 02763_merge 4 14 4 14 4 14 +USING 0 +USING 1 +zzz +==== diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index 0263e1a974f..eabbde9e9dd 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -129,6 +129,27 @@ SELECT x, lc, cnst from merge(currentDatabase(), '02763_merge_fancycols') ORDER SELECT 'SELECT x, y from merge(currentDatabase(), 02763_merge'; SELECT x, y from merge(currentDatabase(), '02763_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + + +CREATE TABLE 02763_t1 ( b String ) ENGINE = MergeTree() ORDER BY tuple(); +INSERT INTO 02763_t1 VALUES('zzz'); +CREATE TABLE 02763_t2 AS 02763_t1 ENGINE = Merge(currentDatabase(), '02763_t1'); + +SELECT 'USING 0'; +CREATE ROW POLICY OR REPLACE 02763_filter_t1 ON 02763_t1 USING 0 TO ALL; +SELECT * FROM 02763_t2 SETTINGS optimize_move_to_prewhere= {{prew}}; + +SELECT 'USING 1'; +CREATE ROW POLICY OR REPLACE 02763_filter_t1 ON 02763_t1 USING 1 TO ALL; +SELECT * FROM 02763_t2 SETTINGS optimize_move_to_prewhere= {{prew}}; + + +DROP TABLE 02763_t1; +DROP TABLE 02763_t2; +SELECT '===='; + + + DROP TABLE 02763_merge_fancycols; DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; @@ -140,4 +161,6 @@ DROP ROW POLICY 02763_filter_4 ON 02763_merge_merge_1; DROP ROW POLICY 02763_filter_5 ON 02763_merge_fancycols; DROP ROW POLICY 02763_filter_6 ON 02763_merge_fancycols; +DROP ROW POLICY 02763_filter_t1 ON 02763_t1; + {% endfor %} From c65261b7376a09c3765267824ada902d1fac446d Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Sat, 30 Mar 2024 20:02:54 +0100 Subject: [PATCH 0782/1165] Update aspell-dict.txt add `hasThreadFuzzer` to aspell-dict.txt --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 57a8e0d5840..1a5f5f219b0 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1673,6 +1673,7 @@ hasSubsequenceCaseInsensitiveUTF hasSubsequenceUTF hasSubstr hasToken +hasThreadFuzzer hasTokenCaseInsensitive hasTokenCaseInsensitiveOrNull hasTokenOrNull From 8f94261e0eb402c109057354ef8073858e7f7b5b Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Sat, 30 Mar 2024 20:04:51 +0100 Subject: [PATCH 0783/1165] Fix spelling mistake insensitve -> insensitive --- docs/en/sql-reference/functions/string-search-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 82226caeba3..e93971879a2 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -521,7 +521,7 @@ Result: ## ngramDistanceCaseInsensitive -Provides a case-insensitve variant of [ngramDistance](#ngramdistance). +Provides a case-insensitive variant of [ngramDistance](#ngramdistance). **Syntax** From b5425c03501ff1cd2bd01d2aaff8a8a3622fe062 Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Sat, 30 Mar 2024 20:07:46 +0100 Subject: [PATCH 0784/1165] Add ngram variants to aspell-dict.txt added the following variants to aspell-dict.txt : ngramDistanceCaseInsensitive ngramDistanceCaseInsensitiveUTF ngramDistanceUTF ngramSearchCaseInsensitive ngramSearchCaseInsensitiveUTF ngramSearchUTF --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 57a8e0d5840..dd13d8a3a6e 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1977,6 +1977,9 @@ nestjs netloc ngram ngramDistance +ngramDistanceCaseInsensitive +ngramDistanceCaseInsensitiveUTF +ngramDistanceUTF ngramMinHash ngramMinHashArg ngramMinHashArgCaseInsensitive @@ -1986,6 +1989,9 @@ ngramMinHashCaseInsensitive ngramMinHashCaseInsensitiveUTF ngramMinHashUTF ngramSearch +ngramSearchCaseInsensitive +ngramSearchCaseInsensitiveUTF +ngramSearchUTF ngramSimHash ngramSimHashCaseInsensitive ngramSimHashCaseInsensitiveUTF From 89cee0a3d6a71e1fbba92ce4546cdb261b196802 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 31 Mar 2024 01:59:36 +0100 Subject: [PATCH 0785/1165] Move bridges to separate packages --- packages/clickhouse-common-static.yaml | 4 --- packages/clickhouse-library-bridge.yaml | 35 ++++++++++++++++++++++ packages/clickhouse-odbc-bridge.yaml | 35 ++++++++++++++++++++++ programs/library-bridge/CMakeLists.txt | 7 +---- programs/odbc-bridge/CMakeLists.txt | 7 +---- programs/odbc-bridge/ColumnInfoHandler.cpp | 1 + programs/odbc-bridge/ColumnInfoHandler.h | 1 - 7 files changed, 73 insertions(+), 17 deletions(-) create mode 100644 packages/clickhouse-library-bridge.yaml create mode 100644 packages/clickhouse-odbc-bridge.yaml diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml index 383ad39591c..db330f808e1 100644 --- a/packages/clickhouse-common-static.yaml +++ b/packages/clickhouse-common-static.yaml @@ -36,10 +36,6 @@ contents: dst: /usr/bin/clickhouse - src: root/usr/bin/clickhouse-extract-from-config dst: /usr/bin/clickhouse-extract-from-config -- src: root/usr/bin/clickhouse-library-bridge - dst: /usr/bin/clickhouse-library-bridge -- src: root/usr/bin/clickhouse-odbc-bridge - dst: /usr/bin/clickhouse-odbc-bridge - src: root/usr/share/bash-completion/completions dst: /usr/share/bash-completion/completions - src: root/usr/share/clickhouse diff --git a/packages/clickhouse-library-bridge.yaml b/packages/clickhouse-library-bridge.yaml new file mode 100644 index 00000000000..95e7d4aaad0 --- /dev/null +++ b/packages/clickhouse-library-bridge.yaml @@ -0,0 +1,35 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-library-bridge" +description: | + ClickHouse Library Bridge - is a separate process for loading libraries for the 'library' dictionary sources and the CatBoost library. + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + +# Common packages config +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" +maintainer: "ClickHouse Dev Team " +deb: + fields: + Source: clickhouse + +# Package specific content +contents: +- src: root/usr/bin/clickhouse-library-bridge + dst: /usr/bin/clickhouse-library-bridge +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-common-static/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-common-static/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-common-static/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-common-static/README.md diff --git a/packages/clickhouse-odbc-bridge.yaml b/packages/clickhouse-odbc-bridge.yaml new file mode 100644 index 00000000000..2a7edf41549 --- /dev/null +++ b/packages/clickhouse-odbc-bridge.yaml @@ -0,0 +1,35 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-odbc-bridge" +description: | + ClickHouse ODBC Bridge - is a separate process for loading ODBC drivers and interacting with external databases using the ODBC protocol. + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + +# Common packages config +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" +maintainer: "ClickHouse Dev Team " +deb: + fields: + Source: clickhouse + +# Package specific content +contents: +- src: root/usr/bin/clickhouse-odbc-bridge + dst: /usr/bin/clickhouse-odbc-bridge +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-common-static/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-common-static/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-common-static/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-common-static/README.md diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt index 98d8848502d..2fca10ce4d7 100644 --- a/programs/library-bridge/CMakeLists.txt +++ b/programs/library-bridge/CMakeLists.txt @@ -24,9 +24,4 @@ target_link_libraries(clickhouse-library-bridge PRIVATE set_target_properties(clickhouse-library-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) -if (SPLIT_DEBUG_SYMBOLS) - clickhouse_split_debug_symbols(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-library-bridge) -else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) - install(TARGETS clickhouse-library-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) -endif() +install(TARGETS clickhouse-library-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index d6cbe8f7215..83839cc21ac 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -30,12 +30,7 @@ target_link_libraries(clickhouse-odbc-bridge PRIVATE set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) target_compile_options (clickhouse-odbc-bridge PRIVATE -Wno-reserved-id-macro -Wno-keyword-macro) -if (SPLIT_DEBUG_SYMBOLS) - clickhouse_split_debug_symbols(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-odbc-bridge) -else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) - install(TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) -endif() +install(TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) if(ENABLE_TESTS) add_subdirectory(tests) diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 4cb15de3b2c..5ff985b3d12 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/programs/odbc-bridge/ColumnInfoHandler.h b/programs/odbc-bridge/ColumnInfoHandler.h index ca7044fdf32..610fb128c9d 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.h +++ b/programs/odbc-bridge/ColumnInfoHandler.h @@ -5,7 +5,6 @@ #if USE_ODBC #include -#include #include #include From e8953a379bc01ef90708d379ce6d95572c04e549 Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Sun, 31 Mar 2024 07:54:58 +0000 Subject: [PATCH 0786/1165] fix external table with Bool Signed-off-by: Duc Canh Le --- src/Core/ExternalTable.cpp | 6 ++++-- .../02935_external_table_enum_type.reference | 6 ++++++ .../0_stateless/02935_external_table_enum_type.sh | 10 ++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index e043a2f9492..f8bbd16d038 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -81,8 +81,10 @@ void BaseExternalTable::parseStructureFromStructureField(const std::string & arg for (auto & child : columns_list_raw->children) { auto * column = child->as(); + /// We use `formatWithPossiblyHidingSensitiveData` instead of `getColumnNameWithoutAlias` because `column->type` is an ASTFunction. + /// `getColumnNameWithoutAlias` will return name of the function with `(arguments)` even if arguments is empty. if (column) - structure.emplace_back(column->name, column->type->getColumnNameWithoutAlias()); + structure.emplace_back(column->name, column->type->formatWithPossiblyHidingSensitiveData(0, true, true)); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: expected column definition, got {}", child->formatForErrorMessage()); } @@ -99,7 +101,7 @@ void BaseExternalTable::parseStructureFromTypesField(const std::string & argumen throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: {}", error); for (size_t i = 0; i < type_list_raw->children.size(); ++i) - structure.emplace_back("_" + toString(i + 1), type_list_raw->children[i]->getColumnNameWithoutAlias()); + structure.emplace_back("_" + toString(i + 1), type_list_raw->children[i]->formatWithPossiblyHidingSensitiveData(0, true, true)); } void BaseExternalTable::initSampleBlock() diff --git a/tests/queries/0_stateless/02935_external_table_enum_type.reference b/tests/queries/0_stateless/02935_external_table_enum_type.reference index ed152e608b9..d13bb1e990c 100644 --- a/tests/queries/0_stateless/02935_external_table_enum_type.reference +++ b/tests/queries/0_stateless/02935_external_table_enum_type.reference @@ -2,3 +2,9 @@ foo 1 bar 2 foo 1 bar 2 +true +true +true +[1] +('foo',1) +(1,1) diff --git a/tests/queries/0_stateless/02935_external_table_enum_type.sh b/tests/queries/0_stateless/02935_external_table_enum_type.sh index 61d5508e9f9..bbbf79fce4a 100755 --- a/tests/queries/0_stateless/02935_external_table_enum_type.sh +++ b/tests/queries/0_stateless/02935_external_table_enum_type.sh @@ -10,3 +10,13 @@ curl -s "${http_url}temp_structure=x+Enum8('foo'%3D1,'bar'%3D2),y+Int" -F "$(pri curl -s "${http_url}temp_types=Enum8('foo'%3D1,'bar'%3D2),Int" -F "$(printf 'temp='"bar"'\t2');filename=data1" -F "query=SELECT * FROM temp" echo -ne 'foo\t1' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Enum8('foo'=1,'bar'=2),y Int" echo -ne 'bar\t2' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --types="Enum8('foo'=1,'bar'=2),Int" + +# https://github.com/ClickHouse/ClickHouse/issues/62108 +echo -ne 'true' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Bool" +echo -ne 'true' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --types="Bool" + +# Test for some complex and custome types +echo -ne 'true' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Nullable(FixedString(4))" +echo -ne '[1]' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Array(UInt8)" +echo -ne '('"'"'foo'"'"',1)' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Tuple(String, UInt8)" +echo -ne '(1,1)' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Point" From 47e865334f4ee0fba5d14fb8938fbcf653cd9c32 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Sun, 31 Mar 2024 13:19:22 +0200 Subject: [PATCH 0787/1165] Fix test with DatabaseReplicated --- .../02908_many_requests_to_system_replicas.reference | 8 ++++---- .../0_stateless/02908_many_requests_to_system_replicas.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference index 0a4008cc35e..f376bb87044 100644 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference @@ -1,8 +1,8 @@ Creating 300 tables -900 2700 2700 -900 2700 2700 -900 2700 2700 -900 2700 2700 +900 1 1 +900 1 1 +900 1 1 +900 1 1 Making 200 requests to system.replicas Query system.replicas while waiting for other concurrent requests to finish 0 diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 18c55159281..144831a2cdc 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -46,10 +46,10 @@ wait; # Check results with different max_block_size -$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas), sum(active_replicas) FROM system.replicas WHERE database=currentDatabase()' -$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas), sum(active_replicas) FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=1' -$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas), sum(active_replicas) FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=77' -$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas), sum(active_replicas) FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=11111' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase()' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=1' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=77' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=11111' echo "Making $CONCURRENCY requests to system.replicas" From 7fb2b7880e51a3c0a36ae3d6249bd7bc76202e45 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 31 Mar 2024 23:05:25 +0200 Subject: [PATCH 0788/1165] Remove useless files --- packages/clickhouse-common-static-dbg.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/clickhouse-common-static-dbg.yaml b/packages/clickhouse-common-static-dbg.yaml index 96de4c17d88..74b7fa8381b 100644 --- a/packages/clickhouse-common-static-dbg.yaml +++ b/packages/clickhouse-common-static-dbg.yaml @@ -30,10 +30,6 @@ conflicts: contents: - src: root/usr/lib/debug/usr/bin/clickhouse.debug dst: /usr/lib/debug/usr/bin/clickhouse.debug -- src: root/usr/lib/debug/usr/bin/clickhouse-odbc-bridge.debug - dst: /usr/lib/debug/usr/bin/clickhouse-odbc-bridge.debug -- src: root/usr/lib/debug/usr/bin/clickhouse-library-bridge.debug - dst: /usr/lib/debug/usr/bin/clickhouse-library-bridge.debug # docs - src: ../AUTHORS dst: /usr/share/doc/clickhouse-common-static-dbg/AUTHORS From 9a9f0161bb94c2800ecf4247a32b3ca2e0c8afa2 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 1 Apr 2024 01:19:00 +0200 Subject: [PATCH 0789/1165] add test --- src/IO/ReadWriteBufferFromHTTP.cpp | 8 +-- .../http_headers_echo_server.py | 37 ++++++++-- .../redirect_server.py | 46 ++++++++++++ .../test_storage_url_http_headers/test.py | 71 ++++++++++++++++--- 4 files changed, 141 insertions(+), 21 deletions(-) create mode 100644 tests/integration/test_storage_url_http_headers/redirect_server.py diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp index 63a91026701..c99b08d0c9d 100644 --- a/src/IO/ReadWriteBufferFromHTTP.cpp +++ b/src/IO/ReadWriteBufferFromHTTP.cpp @@ -346,10 +346,10 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, { if (!mute_logging) LOG_ERROR(log, - "Failed to make request to `{}`{}. " + "Failed to make request to '{}'{}. " "Error: '{}'. " "Failed at try {}/{}.", - initial_uri.toString(), current_uri == initial_uri ? String() : fmt::format(" redirect to `{}`", current_uri.toString()), + initial_uri.toString(), current_uri == initial_uri ? String() : fmt::format(" redirect to '{}'", current_uri.toString()), error_message, attempt, read_settings.http_max_tries); @@ -362,11 +362,11 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, if (!mute_logging) LOG_INFO(log, - "Failed to make request to `{}`{}. " + "Failed to make request to '{}'{}. " "Error: {}. " "Failed at try {}/{}. " "Will retry with current backoff wait is {}/{} ms.", - initial_uri.toString(), current_uri == initial_uri ? String() : fmt::format(" redirect to `{}`", current_uri.toString()), + initial_uri.toString(), current_uri == initial_uri ? String() : fmt::format(" redirect to '{}'", current_uri.toString()), error_message, attempt + 1, read_settings.http_max_tries, milliseconds_to_wait, read_settings.http_retry_max_backoff_ms); diff --git a/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py index b1a3f6777b1..8fb2f8d0e2d 100644 --- a/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py +++ b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py @@ -1,6 +1,8 @@ import http.server +import sys +import json -RESULT_PATH = "/headers.txt" +RESULT_PATH = "/echo_server_headers.txt" class RequestHandler(http.server.BaseHTTPRequestHandler): @@ -8,6 +10,28 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): with open(RESULT_PATH, "w") as f: f.write(self.headers.as_string()) + def do_GET(self): + if self.path == "/": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + if self.path == "/sample-data": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + sample_data = [ + { + "title": "ClickHouse Newsletter June 2022: Materialized, but still real-time", + "theme": "Newsletter", + }, + { + "title": "ClickHouse Over the Years with Benchmarks", + "theme": "ClickHouse Journey", + } + ] + self.wfile.write(bytes(json.dumps(sample_data), "UTF-8")) + def do_POST(self): self.rfile.read1() self.send_response(200) @@ -16,15 +40,16 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): if __name__ == "__main__": - with open(RESULT_PATH, "w") as f: - f.write("") - httpd = http.server.HTTPServer( + host = sys.argv[1] + port = int(sys.argv[2]) + httpd = http.server.ThreadingHTTPServer( ( - "localhost", - 8000, + host, + port, ), RequestHandler, ) + try: httpd.serve_forever() finally: diff --git a/tests/integration/test_storage_url_http_headers/redirect_server.py b/tests/integration/test_storage_url_http_headers/redirect_server.py new file mode 100644 index 00000000000..b1d92d0cd4e --- /dev/null +++ b/tests/integration/test_storage_url_http_headers/redirect_server.py @@ -0,0 +1,46 @@ +import http.server +import sys + +REDIRECT_HOST = "" +REDIRECT_PORT = 0 + +RESULT_PATH = "/redirect_server_headers.txt" + + +class RequestHandler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + with open(RESULT_PATH, "w") as f: + f.write(self.headers.as_string()) + + def do_GET(self): + if self.path == "/": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + else: + global REDIRECT_HOST, REDIRECT_PORT + self.send_response(302) + target_location = f"http://{REDIRECT_HOST}:{REDIRECT_PORT}{self.path}" + self.send_header("Location", target_location) + self.end_headers() + self.wfile.write(b'{"status":"redirected"}') + + +if __name__ == "__main__": + host = sys.argv[1] + port = int(sys.argv[2]) + REDIRECT_HOST = sys.argv[3] + REDIRECT_PORT = int(sys.argv[4]) + httpd = http.server.ThreadingHTTPServer( + ( + host, + port, + ), + RequestHandler, + ) + + try: + httpd.serve_forever() + finally: + httpd.server_close() diff --git a/tests/integration/test_storage_url_http_headers/test.py b/tests/integration/test_storage_url_http_headers/test.py index 3bbf5ec81c9..8fc08ec5c9d 100644 --- a/tests/integration/test_storage_url_http_headers/test.py +++ b/tests/integration/test_storage_url_http_headers/test.py @@ -1,8 +1,7 @@ import pytest import os -import time - from . import http_headers_echo_server +from . import redirect_server from helpers.cluster import ClickHouseCluster @@ -10,31 +9,37 @@ cluster = ClickHouseCluster(__file__) server = cluster.add_instance("node") -def run_echo_server(): +def run_server(container_id, file_name, hostname, port, *args): script_dir = os.path.dirname(os.path.realpath(__file__)) - server.copy_file_to_container( - os.path.join(script_dir, "http_headers_echo_server.py"), - "/http_headers_echo_server.py", + cluster.copy_file_to_container( + container_id, + os.path.join(script_dir, file_name), + f"/{file_name}", ) - server.exec_in_container( + cmd_args = [hostname, port] + list(args) + cmd_args_val = " ".join([str(x) for x in cmd_args]) + + cluster.exec_in_container( + container_id, [ "bash", "-c", - "python3 /http_headers_echo_server.py > /http_headers_echo.server.log 2>&1", + f"python3 /{file_name} {cmd_args_val} > {file_name}.log 2>&1", ], detach=True, user="root", ) for _ in range(0, 10): - ping_response = server.exec_in_container( - ["curl", "-s", f"http://localhost:8000/"], + ping_response = cluster.exec_in_container( + container_id, + ["curl", "-s", f"http://{hostname}:{port}/"], nothrow=True, ) - if "html" in ping_response: + if '{"status":"ok"}' in ping_response: return print(ping_response) @@ -42,11 +47,23 @@ def run_echo_server(): raise Exception("Echo server is not responding") +def run_echo_server(): + container_id = cluster.get_container_id("node") + run_server(container_id, "http_headers_echo_server.py", "localhost", 8000) + + +def run_redirect_server(): + container_id = cluster.get_container_id("node") + run_server(container_id, "redirect_server.py", "localhost", 8080, "localhost", 8000) + + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() + run_redirect_server() run_echo_server() + yield cluster finally: cluster.shutdown() @@ -64,3 +81,35 @@ def test_storage_url_http_headers(started_cluster): print(result) assert "X-My-Custom-Header: test-header" in result + + +def test_storage_url_redirected_headers(started_cluster): + query = """ + SELECT + title::String as title, + theme::String as theme + FROM + url('http://127.0.0.1:8080/sample-data', 'JSONEachRow', 'title String, theme String') + SETTINGS http_max_tries=2, max_http_get_redirects=2 + """ + + result = server.query(query) + assert 2 == len(result.strip().split("\n")) + + result_redirect = server.exec_in_container( + ["cat", redirect_server.RESULT_PATH], user="root" + ) + + print(result_redirect) + + assert "Host: 127.0.0.1" in result_redirect + assert "Host: localhost" not in result_redirect + + result = server.exec_in_container( + ["cat", http_headers_echo_server.RESULT_PATH], user="root" + ) + + print(result) + + assert "Host: 127.0.0.1" not in result + assert "Host: localhost" in result From 267ebba3c0a654da36aac5146012dd4b162a34bc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 31 Mar 2024 23:43:28 +0000 Subject: [PATCH 0790/1165] Automatic style fix --- .../test_storage_url_http_headers/http_headers_echo_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py index 8fb2f8d0e2d..3c62112a7d3 100644 --- a/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py +++ b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py @@ -28,8 +28,8 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): { "title": "ClickHouse Over the Years with Benchmarks", "theme": "ClickHouse Journey", - } - ] + }, + ] self.wfile.write(bytes(json.dumps(sample_data), "UTF-8")) def do_POST(self): From ff11d67f85cdf35e06e2fdd6d3a3409339f5e570 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Mon, 1 Apr 2024 10:50:02 +0800 Subject: [PATCH 0791/1165] add tests --- src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp index f8652a430df..117d98d23bb 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp @@ -15,6 +15,7 @@ static FormatSettings updateFormatSettings(const FormatSettings & settings, cons { FormatSettings updated = settings; updated.skip_unknown_fields = true; + updated.with_names_use_header = false; updated.date_time_input_format = FormatSettings::DateTimeInputFormat::BestEffort; updated.defaults_for_omitted_fields = true; updated.csv.delimiter = updated.hive_text.fields_delimiter; @@ -33,7 +34,7 @@ HiveTextRowInputFormat::HiveTextRowInputFormat( HiveTextRowInputFormat::HiveTextRowInputFormat( const Block & header_, std::shared_ptr buf_, const Params & params_, const FormatSettings & format_settings_) : CSVRowInputFormat( - header_, buf_, params_, true, false, format_settings_, std::make_unique(*buf_, format_settings_)) + header_, buf_, params_, false, false, format_settings_, std::make_unique(*buf_, format_settings_)) { } From e4473eb969df3296faf57efc9deafb02d19b479e Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Mon, 1 Apr 2024 11:51:14 +0800 Subject: [PATCH 0792/1165] add tests --- .../03033_hive_text_read_variable_fields.reference | 2 ++ .../03033_hive_text_read_variable_fields.sh | 13 +++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/queries/0_stateless/03033_hive_text_read_variable_fields.reference create mode 100755 tests/queries/0_stateless/03033_hive_text_read_variable_fields.sh diff --git a/tests/queries/0_stateless/03033_hive_text_read_variable_fields.reference b/tests/queries/0_stateless/03033_hive_text_read_variable_fields.reference new file mode 100644 index 00000000000..2e7c474620b --- /dev/null +++ b/tests/queries/0_stateless/03033_hive_text_read_variable_fields.reference @@ -0,0 +1,2 @@ +1 3 0 +3 5 9 diff --git a/tests/queries/0_stateless/03033_hive_text_read_variable_fields.sh b/tests/queries/0_stateless/03033_hive_text_read_variable_fields.sh new file mode 100755 index 00000000000..5af0e465cca --- /dev/null +++ b/tests/queries/0_stateless/03033_hive_text_read_variable_fields.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# NOTE: this sh wrapper is required because of shell_config + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "drop table if exists test_tbl" +$CLICKHOUSE_CLIENT -q "create table test_tbl (a UInt16, b UInt32, c UInt32) engine=MergeTree order by a" +$CLICKHOUSE_CLIENT -q "insert into test_tbl from infile '$CURDIR/data_hive/fields_number_variable.txt' SETTINGS input_format_hive_text_fields_delimiter=',' FORMAT HIVETEXT" +$CLICKHOUSE_CLIENT -q "select * from test_tbl" +$CLICKHOUSE_CLIENT -q "drop table test_tbl" \ No newline at end of file From 9b1b81d92120fa6251864f27762918a85c7c705b Mon Sep 17 00:00:00 2001 From: zhongyuankai <872237106@qq.com> Date: Mon, 1 Apr 2024 11:49:25 +0800 Subject: [PATCH 0793/1165] add tests --- src/Storages/StorageMemory.cpp | 34 +++++++++++ ...2_storage_memory_modify_settings.reference | 24 ++++---- .../03032_storage_memory_modify_settings.sql | 58 ++++++++++--------- 3 files changed, 79 insertions(+), 37 deletions(-) diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 055d9ee6fa1..c5feb6dad21 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -307,6 +307,40 @@ void StorageMemory::alter(const DB::AlterCommands & params, DB::ContextPtr conte auto copy = memory_settings; copy.applyChanges(settings_changes.changes); copy.sanityCheck(); + + /// When modifying the values of max_bytes_to_keep and max_rows_to_keep to be smaller than the old values, + /// the old data needs to be removed. + if (!memory_settings.max_bytes_to_keep || memory_settings.max_bytes_to_keep > copy.max_bytes_to_keep + || !memory_settings.max_rows_to_keep || memory_settings.max_rows_to_keep > copy.max_rows_to_keep) + { + std::lock_guard lock(mutex); + + auto new_data = std::make_unique(*(data.get())); + UInt64 new_total_rows = total_size_rows.load(std::memory_order_relaxed); + UInt64 new_total_bytes = total_size_bytes.load(std::memory_order_relaxed); + while (!new_data->empty() + && ((copy.max_bytes_to_keep && new_total_bytes > copy.max_bytes_to_keep) + || (copy.max_rows_to_keep && new_total_rows > copy.max_rows_to_keep))) + { + Block oldest_block = new_data->front(); + UInt64 rows_to_remove = oldest_block.rows(); + UInt64 bytes_to_remove = oldest_block.allocatedBytes(); + if (new_total_bytes - bytes_to_remove < copy.min_bytes_to_keep + || new_total_rows - rows_to_remove < copy.min_rows_to_keep) + { + break; // stop - removing next block will put us under min_bytes / min_rows threshold + } + + // delete old block from current storage table + new_total_rows -= rows_to_remove; + new_total_bytes -= bytes_to_remove; + new_data->erase(new_data->begin()); + } + + data.set(std::move(new_data)); + total_size_rows.store(new_total_rows, std::memory_order_relaxed); + total_size_bytes.store(new_total_bytes, std::memory_order_relaxed); + } memory_settings = std::move(copy); } diff --git a/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference b/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference index 20dda4fa15a..f7d25c40a43 100644 --- a/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference +++ b/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference @@ -1,16 +1,20 @@ -TESTING BYTES -8192 -9216 -9216 +TESTING MODIFY SMALLER BYTES +17408 +16384 65536 -TESTING ROWS +TESTING MODIFY SMALLER ROWS +9216 +8192 +4096 +TESTING ADD SETTINGS +50 +1000 +1070 +1020 +1100 +TESTING ADD SETTINGS 50 1000 1020 1100 -TESTING NO CIRCULAR-BUFFER -8192 -9216 -17408 -82944 TESTING INVALID SETTINGS diff --git a/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql index 34be327175e..bfa13ee0ec8 100644 --- a/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql +++ b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql @@ -1,64 +1,68 @@ SET max_block_size = 65409; -- Default value +SELECT 'TESTING MODIFY SMALLER BYTES'; DROP TABLE IF EXISTS memory; -CREATE TABLE memory (i UInt32) ENGINE = Memory; +CREATE TABLE memory (i UInt32) ENGINE = Memory SETTINGS min_bytes_to_keep = 8192, max_bytes_to_keep = 32768; + +INSERT INTO memory SELECT * FROM numbers(0, 100); +INSERT INTO memory SELECT * FROM numbers(0, 3000); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); ALTER TABLE memory MODIFY SETTING min_bytes_to_keep = 4096, max_bytes_to_keep = 16384; - -SELECT 'TESTING BYTES'; -/* 1. testing oldest block doesn't get deleted because of min-threshold */ -INSERT INTO memory SELECT * FROM numbers(0, 1600); SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -/* 2. adding block that doesn't get deleted */ -INSERT INTO memory SELECT * FROM numbers(1000, 100); +INSERT INTO memory SELECT * FROM numbers(3000, 10000); SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -/* 3. testing oldest block gets deleted - 9216 bytes - 1100 */ -INSERT INTO memory SELECT * FROM numbers(9000, 1000); +SELECT 'TESTING MODIFY SMALLER ROWS'; +DROP TABLE IF EXISTS memory; +CREATE TABLE memory (i UInt32) ENGINE = Memory SETTINGS min_rows_to_keep = 200, max_rows_to_keep = 2000; + +INSERT INTO memory SELECT * FROM numbers(0, 100); +INSERT INTO memory SELECT * FROM numbers(100, 1000); SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -/* 4.check large block over-writes all bytes / rows */ -INSERT INTO memory SELECT * FROM numbers(9000, 10000); +ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +INSERT INTO memory SELECT * FROM numbers(1000, 500); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + +SELECT 'TESTING ADD SETTINGS'; DROP TABLE IF EXISTS memory; CREATE TABLE memory (i UInt32) ENGINE = Memory; -ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; - -SELECT 'TESTING ROWS'; -/* 1. add normal number of rows */ INSERT INTO memory SELECT * FROM numbers(0, 50); SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -/* 2. table should have 1000 */ INSERT INTO memory SELECT * FROM numbers(50, 950); SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -/* 3. table should have 1020 - removed first 50 */ INSERT INTO memory SELECT * FROM numbers(2000, 70); SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -/* 4. check large block over-writes all rows */ +ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); + INSERT INTO memory SELECT * FROM numbers(3000, 1100); SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -SELECT 'TESTING NO CIRCULAR-BUFFER'; +SELECT 'TESTING ADD SETTINGS'; DROP TABLE IF EXISTS memory; CREATE TABLE memory (i UInt32) ENGINE = Memory; +ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; -INSERT INTO memory SELECT * FROM numbers(0, 1600); -SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +INSERT INTO memory SELECT * FROM numbers(0, 50); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(1000, 100); -SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +INSERT INTO memory SELECT * FROM numbers(50, 950); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(9000, 1000); -SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +INSERT INTO memory SELECT * FROM numbers(2000, 70); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(9000, 10000); -SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +INSERT INTO memory SELECT * FROM numbers(3000, 1100); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); SELECT 'TESTING INVALID SETTINGS'; DROP TABLE IF EXISTS memory; From 9a37552af6b0c78c2fcaa9dd35f51e2ef8aeeb22 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Mon, 1 Apr 2024 12:05:41 +0800 Subject: [PATCH 0794/1165] add test file --- tests/queries/0_stateless/data_hive/fields_number_variable.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/data_hive/fields_number_variable.txt diff --git a/tests/queries/0_stateless/data_hive/fields_number_variable.txt b/tests/queries/0_stateless/data_hive/fields_number_variable.txt new file mode 100644 index 00000000000..b4e037978b9 --- /dev/null +++ b/tests/queries/0_stateless/data_hive/fields_number_variable.txt @@ -0,0 +1,2 @@ +1,3 +3,5,9 \ No newline at end of file From 724044c15ed7351fc5aaddbd4a96f53142f593dc Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Mon, 1 Apr 2024 18:22:17 +0800 Subject: [PATCH 0795/1165] ut fix --- src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp index 117d98d23bb..49495503455 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp @@ -15,7 +15,7 @@ static FormatSettings updateFormatSettings(const FormatSettings & settings, cons { FormatSettings updated = settings; updated.skip_unknown_fields = true; - updated.with_names_use_header = false; + updated.with_names_use_header = true; updated.date_time_input_format = FormatSettings::DateTimeInputFormat::BestEffort; updated.defaults_for_omitted_fields = true; updated.csv.delimiter = updated.hive_text.fields_delimiter; @@ -34,7 +34,7 @@ HiveTextRowInputFormat::HiveTextRowInputFormat( HiveTextRowInputFormat::HiveTextRowInputFormat( const Block & header_, std::shared_ptr buf_, const Params & params_, const FormatSettings & format_settings_) : CSVRowInputFormat( - header_, buf_, params_, false, false, format_settings_, std::make_unique(*buf_, format_settings_)) + header_, buf_, params_, true, false, format_settings_, std::make_unique(*buf_, header_, format_settings_)) { } @@ -45,7 +45,8 @@ HiveTextFormatReader::HiveTextFormatReader(PeekableReadBuffer & buf_, const Form std::vector HiveTextFormatReader::readNames() { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "HiveTextRowInputFormat::readNames is not implemented"); + PeekableReadBufferCheckpoint checkpoint{*buf, true}; + return input_field_names; } std::vector HiveTextFormatReader::readTypes() From 2b4e6439ef6c5746085e39c9224024bf2f643a8d Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Mon, 1 Apr 2024 18:29:32 +0800 Subject: [PATCH 0796/1165] remove useless code --- src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp index 49495503455..6ddba781a54 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp @@ -34,7 +34,7 @@ HiveTextRowInputFormat::HiveTextRowInputFormat( HiveTextRowInputFormat::HiveTextRowInputFormat( const Block & header_, std::shared_ptr buf_, const Params & params_, const FormatSettings & format_settings_) : CSVRowInputFormat( - header_, buf_, params_, true, false, format_settings_, std::make_unique(*buf_, header_, format_settings_)) + header_, buf_, params_, true, false, format_settings_, std::make_unique(*buf_, format_settings_)) { } From b2e764f63326e22d3cb306bd0009108b1a955934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 1 Apr 2024 13:51:56 +0200 Subject: [PATCH 0797/1165] Revert "Merge pull request #61564 from liuneng1994/optimize_in_single_value" This reverts commit 20a45b40735e4b033bdcd0b5ea6fe4585e4f840c, reversing changes made to a642f4d3ec3f28edb2def71bbf7ac2bf211b5bac. --- src/Analyzer/Passes/ConvertInToEqualPass.cpp | 75 ------- src/Analyzer/Passes/ConvertInToEqualPass.h | 27 --- src/Analyzer/QueryTreePassManager.cpp | 2 - src/Functions/CMakeLists.txt | 2 - src/Functions/equals.cpp | 5 - src/Functions/equals.h | 11 - src/Functions/notEquals.cpp | 5 - src/Functions/notEquals.h | 11 - ..._transform_query_for_external_database.cpp | 6 +- tests/performance/function_in.xml | 28 --- .../03013_optimize_in_to_equal.reference | 189 ------------------ .../03013_optimize_in_to_equal.sql | 32 --- 12 files changed, 2 insertions(+), 391 deletions(-) delete mode 100644 src/Analyzer/Passes/ConvertInToEqualPass.cpp delete mode 100644 src/Analyzer/Passes/ConvertInToEqualPass.h delete mode 100644 src/Functions/equals.h delete mode 100644 src/Functions/notEquals.h delete mode 100644 tests/performance/function_in.xml delete mode 100644 tests/queries/0_stateless/03013_optimize_in_to_equal.reference delete mode 100644 tests/queries/0_stateless/03013_optimize_in_to_equal.sql diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp deleted file mode 100644 index b204d2fb922..00000000000 --- a/src/Analyzer/Passes/ConvertInToEqualPass.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -class ConvertInToEqualPassVisitor : public InDepthQueryTreeVisitorWithContext -{ -public: - using Base = InDepthQueryTreeVisitorWithContext; - using Base::Base; - - void enterImpl(QueryTreeNodePtr & node) - { - static const std::unordered_map MAPPING = { - {"in", "equals"}, - {"notIn", "notEquals"} - }; - auto * func_node = node->as(); - if (!func_node - || !MAPPING.contains(func_node->getFunctionName()) - || func_node->getArguments().getNodes().size() != 2) - return ; - auto args = func_node->getArguments().getNodes(); - auto * column_node = args[0]->as(); - auto * constant_node = args[1]->as(); - if (!column_node || !constant_node) - return ; - // IN multiple values is not supported - if (constant_node->getValue().getType() == Field::Types::Which::Tuple - || constant_node->getValue().getType() == Field::Types::Which::Array) - return ; - // x IN null not equivalent to x = null - if (constant_node->getValue().isNull()) - return ; - auto result_func_name = MAPPING.at(func_node->getFunctionName()); - auto equal = std::make_shared(result_func_name); - auto new_const = std::make_shared(constant_node->getValue(), removeNullable(constant_node->getResultType())); - new_const->getSourceExpression() = constant_node->getSourceExpression(); - QueryTreeNodes arguments{column_node->clone(), new_const}; - equal->getArguments().getNodes() = std::move(arguments); - FunctionOverloadResolverPtr resolver; - bool decimal_check_overflow = getContext()->getSettingsRef().decimal_check_overflow; - if (result_func_name == "equals") - { - resolver = createInternalFunctionEqualOverloadResolver(decimal_check_overflow); - } - else - { - resolver = createInternalFunctionNotEqualOverloadResolver(decimal_check_overflow); - } - try - { - equal->resolveAsFunction(resolver); - } - catch (...) - { - // When function resolver fails, we should not replace the function node - return; - } - node = equal; - } -}; - -void ConvertInToEqualPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) -{ - ConvertInToEqualPassVisitor visitor(std::move(context)); - visitor.visit(query_tree_node); -} -} diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.h b/src/Analyzer/Passes/ConvertInToEqualPass.h deleted file mode 100644 index bd4f8607c88..00000000000 --- a/src/Analyzer/Passes/ConvertInToEqualPass.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include - -namespace DB -{ -/** Optimize `in` to `equals` if possible. - * 1. convert in single value to equal - * Example: SELECT * from test where x IN (1); - * Result: SELECT * from test where x = 1; - * - * 2. convert not in single value to notEqual - * Example: SELECT * from test where x NOT IN (1); - * Result: SELECT * from test where x != 1; - * - * If value is null or tuple, do not convert. - */ -class ConvertInToEqualPass final : public IQueryTreePass -{ -public: - String getName() override { return "ConvertInToEqualPass"; } - - String getDescription() override { return "Convert in to equal"; } - - void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; -}; -} diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 14eb179680c..9c07884a464 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -264,7 +263,6 @@ void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze) manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); - manager.addPass(std::make_unique()); /// should before AggregateFunctionsArithmericOperationsPass manager.addPass(std::make_unique()); diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index d5eb12f3dee..733ae25274e 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -14,8 +14,6 @@ extract_into_parent_list(clickhouse_functions_sources dbms_sources multiMatchAny.cpp checkHyperscanRegexp.cpp array/has.cpp - equals.cpp - notEquals.cpp CastOverloadResolver.cpp ) extract_into_parent_list(clickhouse_functions_headers dbms_headers diff --git a/src/Functions/equals.cpp b/src/Functions/equals.cpp index 512abaa6fc7..5c59daf0537 100644 --- a/src/Functions/equals.cpp +++ b/src/Functions/equals.cpp @@ -13,11 +13,6 @@ REGISTER_FUNCTION(Equals) factory.registerFunction(); } -FunctionOverloadResolverPtr createInternalFunctionEqualOverloadResolver(bool decimal_check_overflow) -{ - return std::make_unique(std::make_shared(decimal_check_overflow)); -} - template <> ColumnPtr FunctionComparison::executeTupleImpl( const ColumnsWithTypeAndName & x, const ColumnsWithTypeAndName & y, size_t tuple_size, size_t input_rows_count) const diff --git a/src/Functions/equals.h b/src/Functions/equals.h deleted file mode 100644 index 855cba4db3e..00000000000 --- a/src/Functions/equals.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once -#include - -namespace DB -{ - -class IFunctionOverloadResolver; -using FunctionOverloadResolverPtr = std::shared_ptr; - -FunctionOverloadResolverPtr createInternalFunctionEqualOverloadResolver(bool decimal_check_overflow); -} diff --git a/src/Functions/notEquals.cpp b/src/Functions/notEquals.cpp index 744a0997d95..3a63db46711 100644 --- a/src/Functions/notEquals.cpp +++ b/src/Functions/notEquals.cpp @@ -12,11 +12,6 @@ REGISTER_FUNCTION(NotEquals) factory.registerFunction(); } -FunctionOverloadResolverPtr createInternalFunctionNotEqualOverloadResolver(bool decimal_check_overflow) -{ - return std::make_unique(std::make_shared(decimal_check_overflow)); -} - template <> ColumnPtr FunctionComparison::executeTupleImpl( const ColumnsWithTypeAndName & x, const ColumnsWithTypeAndName & y, size_t tuple_size, size_t input_rows_count) const diff --git a/src/Functions/notEquals.h b/src/Functions/notEquals.h deleted file mode 100644 index 961889d68d7..00000000000 --- a/src/Functions/notEquals.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once -#include - -namespace DB -{ - -class IFunctionOverloadResolver; -using FunctionOverloadResolverPtr = std::shared_ptr; - -FunctionOverloadResolverPtr createInternalFunctionNotEqualOverloadResolver(bool decimal_check_overflow); -} diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 6490498d717..7e2d393c3d1 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -306,8 +306,7 @@ TEST(TransformQueryForExternalDatabase, Aliases) check(state, 1, {"field"}, "SELECT field AS value, field AS display FROM table WHERE field NOT IN ('') AND display LIKE '%test%'", - R"(SELECT "field" FROM "test"."table" WHERE ("field" NOT IN ('')) AND ("field" LIKE '%test%'))", - R"(SELECT "field" FROM "test"."table" WHERE ("field" != '') AND ("field" LIKE '%test%'))"); + R"(SELECT "field" FROM "test"."table" WHERE ("field" NOT IN ('')) AND ("field" LIKE '%test%'))"); } TEST(TransformQueryForExternalDatabase, ForeignColumnInWhere) @@ -409,6 +408,5 @@ TEST(TransformQueryForExternalDatabase, Analyzer) check(state, 1, {"column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo"}, "SELECT * FROM table WHERE (column) IN (1)", - R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" IN (1))", - R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" = 1)"); + R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" IN (1))"); } diff --git a/tests/performance/function_in.xml b/tests/performance/function_in.xml deleted file mode 100644 index af4f8737ba7..00000000000 --- a/tests/performance/function_in.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - 8 - 1 - - - - CREATE TABLE t_nullable - ( - key_string1 Nullable(String), - key_string2 Nullable(String), - key_string3 Nullable(String), - key_int64_1 Nullable(Int64), - key_int64_2 Nullable(Int64), - key_int64_3 Nullable(Int64), - key_int64_4 Nullable(Int64), - key_int64_5 Nullable(Int64), - m1 Int64, - m2 Int64 - ) - ENGINE = Memory - - insert into t_nullable select ['aaaaaa','bbaaaa','ccaaaa','ddaaaa'][number % 101 + 1], ['aa','bb','cc','dd'][number % 100 + 1], ['aa','bb','cc','dd'][number % 102 + 1], number%10+1, number%10+2, number%10+3, number%10+4,number%10+5, number%6000+1, number%5000+2 from numbers_mt(30000000) - select * from t_nullable where key_string1 in ('aaaaaa') format Null SETTINGS allow_experimental_analyzer=1 - select * from t_nullable where key_string2 in ('3') format Null SETTINGS allow_experimental_analyzer=1 - drop table if exists t_nullable - - diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.reference b/tests/queries/0_stateless/03013_optimize_in_to_equal.reference deleted file mode 100644 index 7d1118b7730..00000000000 --- a/tests/queries/0_stateless/03013_optimize_in_to_equal.reference +++ /dev/null @@ -1,189 +0,0 @@ -a 1 -------------------- -0 -0 -0 -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: equals, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: \'a\', constant_value_type: String -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: equals, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: \'A\', constant_value_type: String - EXPRESSION - FUNCTION id: 9, function_name: upper, function_type: ordinary, result_type: String - ARGUMENTS - LIST id: 10, nodes: 1 - CONSTANT id: 11, constant_value: \'a\', constant_value_type: String -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: in, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: Tuple_(\'a\', \'b\'), constant_value_type: Tuple(String, String) -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: in, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: Array_[\'a\', \'b\'], constant_value_type: Array(String) -------------------- -b 2 -c 3 -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: notEquals, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: \'a\', constant_value_type: String -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: notEquals, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: \'A\', constant_value_type: String - EXPRESSION - FUNCTION id: 9, function_name: upper, function_type: ordinary, result_type: String - ARGUMENTS - LIST id: 10, nodes: 1 - CONSTANT id: 11, constant_value: \'a\', constant_value_type: String -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: notIn, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: Tuple_(\'a\', \'b\'), constant_value_type: Tuple(String, String) -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: notIn, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: Array_[\'a\', \'b\'], constant_value_type: Array(String) -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: notIn, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: NULL, constant_value_type: Nullable(Nothing) -------------------- -QUERY id: 0 - PROJECTION COLUMNS - x String - y Int32 - PROJECTION - LIST id: 1, nodes: 2 - COLUMN id: 2, column_name: x, result_type: String, source_id: 3 - COLUMN id: 4, column_name: y, result_type: Int32, source_id: 3 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.test - WHERE - FUNCTION id: 5, function_name: in, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 6, nodes: 2 - COLUMN id: 7, column_name: x, result_type: String, source_id: 3 - CONSTANT id: 8, constant_value: NULL, constant_value_type: Nullable(Nothing) -------------------- diff --git a/tests/queries/0_stateless/03013_optimize_in_to_equal.sql b/tests/queries/0_stateless/03013_optimize_in_to_equal.sql deleted file mode 100644 index e0eaa84cb8d..00000000000 --- a/tests/queries/0_stateless/03013_optimize_in_to_equal.sql +++ /dev/null @@ -1,32 +0,0 @@ -DROP TABLE IF EXISTS test; -CREATE TABLE test (x String, y Int32) ENGINE = MergeTree() ORDER BY x; -SET allow_experimental_analyzer = 1; -INSERT INTO test VALUES ('a', 1), ('b', 2), ('c', 3); -select * from test where x in ('a'); -select '-------------------'; -select x in Null from test; -select '-------------------'; -explain query tree select * from test where x in ('a'); -select '-------------------'; -explain query tree select * from test where x in (upper('a')); -select '-------------------'; -explain query tree select * from test where x in ('a','b'); -select '-------------------'; -explain query tree select * from test where x in ['a','b']; -select '-------------------'; -select * from test where x not in ('a'); -select '-------------------'; -explain query tree select * from test where x not in ('a'); -select '-------------------'; -explain query tree select * from test where x not in (upper('a')); -select '-------------------'; -explain query tree select * from test where x not in ('a','b'); -select '-------------------'; -explain query tree select * from test where x not in ['a','b']; -select '-------------------'; -explain query tree select * from test where x not in (NULL); -select '-------------------'; -explain query tree select * from test where x in (NULL); -select '-------------------'; ---- fuzzed -SELECT number FROM numbers(2) WHERE arrayExists(_ -> (_ IN toNullable(4294967290)), [number]); From a7f48cf936a8f50388ef9a2ca49827246499e62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 1 Apr 2024 14:12:36 +0200 Subject: [PATCH 0798/1165] Fix upgrade check --- src/Core/SettingsChangesHistory.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 170836cb980..768b6aa6cbd 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -85,8 +85,9 @@ namespace SettingsChangesHistory /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) static std::map settings_changes_history = { + {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, + }}, {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, - {"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"}, {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"}, From 5e9436611aa0358629e50cf6ef8749936149978a Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 1 Apr 2024 10:21:23 -0300 Subject: [PATCH 0799/1165] fix incorrect ci err message --- tests/clickhouse-test | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 10851d23481..0661380e061 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -69,6 +69,7 @@ TEST_FILE_EXTENSIONS = [".sql", ".sql.j2", ".sh", ".py", ".expect"] VERSION_PATTERN = r"^((\d+\.)?(\d+\.)?(\d+\.)?\d+)$" +TEST_MAX_RUN_TIME_IN_SECONDS = 120 class SharedEngineReplacer: ENGINES_NON_REPLICATED_REGEXP = r"[ =]((Collapsing|VersionedCollapsing|Summing|Replacing|Aggregating|)MergeTree\(?\)?)" @@ -682,7 +683,7 @@ class FailureReason(enum.Enum): STDERR = "having stderror: " EXCEPTION = "having exception in stdout: " RESULT_DIFF = "result differs with reference: " - TOO_LONG = "Test runs too long (> 60s). Make it faster." + TOO_LONG = f"Test runs too long (> {TEST_MAX_RUN_TIME_IN_SECONDS}s). Make it faster." INTERNAL_QUERY_FAIL = "Internal query (CREATE/DROP DATABASE) failed:" # SKIPPED reasons @@ -1421,7 +1422,7 @@ class TestCase: if ( self.testcase_args.test_runs > 1 - and total_time > 120 + and total_time > TEST_MAX_RUN_TIME_IN_SECONDS and "long" not in self.tags ): if debug_log: From fe8d4b5dfc659682eeb70b285809377eb71f4719 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Mon, 1 Apr 2024 17:06:54 +0200 Subject: [PATCH 0800/1165] refine --- .../functions/string-search-functions.md | 8 +-- .../functions/string-search-functions.md | 53 ++++++++++--------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 1b03f220db2..53a6e1de4a4 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -74,6 +74,8 @@ SELECT position('Hello, world!', 'o', 7) ``` +Result: + ``` text ┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐ │ 5 │ 9 │ @@ -491,7 +493,7 @@ ngramDistance(haystack, needle) ## ngramSearch -Like `ngramDistance` but calculates the non-symmetric difference between a `needle` string and a `haystack` string, i.e. the number of n-grams from needle minus the common number of n-grams normalized by the number of `needle` n-grams. Returns a Float32 between 0 and 1. The bigger the result is, the more likely `needle` is in the `haystack`. This function is useful for fuzzy string search. Also see function `soundex`. +Like `ngramDistance` but calculates the non-symmetric difference between a `needle` string and a `haystack` string, i.e. the number of n-grams from `needle` minus the common number of n-grams normalized by the number of `needle` n-grams. Returns a Float32 between 0 and 1. The bigger the result is, the more likely `needle` is in the `haystack`. This function is useful for fuzzy string search. Also see function `soundex`. Functions `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. @@ -610,7 +612,7 @@ Like `countMatches(haystack, pattern)` but matching ignores the case. ## regexpExtract -Extracts the first string in haystack that matches the regexp pattern and corresponds to the regex group index. +Extracts the first string in `haystack` that matches the regexp pattern and corresponds to the regex group index. **Syntax** @@ -652,7 +654,7 @@ Result: ## hasSubsequence -Returns 1 if needle is a subsequence of haystack, or 0 otherwise. +Returns 1 if `needle` is a subsequence of `haystack`, or 0 otherwise. A subsequence of a string is a sequence that can be derived from the given string by deleting zero or more elements without changing the order of the remaining elements. diff --git a/docs/zh/sql-reference/functions/string-search-functions.md b/docs/zh/sql-reference/functions/string-search-functions.md index 14c40fa1243..3d88704a297 100644 --- a/docs/zh/sql-reference/functions/string-search-functions.md +++ b/docs/zh/sql-reference/functions/string-search-functions.md @@ -9,7 +9,7 @@ slug: /zh/sql-reference/functions/string-search-functions 例如。英语中大写的`i`是`I`，而在土耳其语中则是`İ`, 对于英语以外的语言，结果可能会不符合预期。 本节中的函数还假设搜索字符串和被搜索字符串是单字节编码文本(例如ASCII)。如果违反此假设，不会抛出异常且结果为undefined。 -UTF-8 编码字符串的搜索通常由单独的函数变体提供。同样，如果使用 UTF-8 函数变体但输入字符串不是 UTF-8 编码文本，不会抛出异常且结果为undefined。 +UTF-8 编码字符串的搜索通常由单独的函数变体提供。同样，如果使用 UTF-8 函数变体但输入字符串不是 UTF-8 编码文本，不会抛出异常且结果为 undefined。 需要注意，函数不会执行自动 Unicode 规范化，您可以使用[normalizeUTF8*()](https://clickhouse.com/docs/zh/sql-reference/functions/string-functions/) 函数来执行此操作。 在[字符串函数](string-functions.md) 和 [字符串替换函数](string-replace-functions.md) 会分别说明. @@ -29,17 +29,17 @@ position(haystack, needle[, start_pos]) **参数** - `haystack` — 被检索查询字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — 子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – 在字符串`haystack` 中开始检索的位置(从1开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选 +- `needle` — 进行查询的子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` – 在字符串`haystack` 中开始检索的位置(从1开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选。 **返回值** - 若子字符串存在，返回位置(以字节为单位，从 1 开始）。 -- 如果不存在子字符串,返回0。 +- 如果不存在子字符串，返回 0。 如果子字符串 `needle` 为空，则： - 如果未指定 `start_pos`，返回 `1` -- 如果 `start_pos` 为 0，则返回 `1` +- 如果 `start_pos = 0`，则返回 `1` - 如果 `start_pos >= 1` 且 `start_pos <= length(haystack) + 1`，则返回 `start_pos` - 否则返回 `0` @@ -68,14 +68,16 @@ SELECT position('Hello, world!', 'o', 1), position('Hello, world!', 'o', 7) ``` + 结果： + ``` text ┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐ │ 5 │ 9 │ └───────────────────────────────────┴───────────────────────────────────┘ ``` -示例，语法别名 `needle IN haystack`: +示例，`needle IN haystack`: ```sql SELECT 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); @@ -154,11 +156,11 @@ SELECT positionUTF8('Motörhead', 'r'); ## multiSearchAllPositions -类似于 [position](#position)返回多个在字符串 `haystack` 中 `needle` 子字符串的位置的数组（以字节为单位，从 1 开始）。 +类似于 [position](#position) 但是返回多个在字符串 `haystack` 中 `needle` 子字符串的位置的数组（以字节为单位，从 1 开始）。 :::note -所有以 `multiSearch*()` 开头的函数最多支持28 个`needle`. +所有以 `multiSearch*()` 开头的函数仅支持最多 28 个`needle`. ::: **语法** @@ -208,7 +210,7 @@ multiSearchFirstPosition(haystack, [needle1, needle2, …, needleN]) ## multiSearchFirstIndex -在字符串`haystack`中匹配多个`needle`子字符串，从左开始任一匹配的子串，返回其索引 `i` (从1开始)，如无法匹配则返回0。 +在字符串`haystack`中匹配最左侧的`needle`子字符串i，返回其索引 `i` (从1开始)，如无法匹配则返回0。 函数 `multiSearchFirstIndexCaseInsensitive`, `multiSearchFirstIndexUTF8` 和 `multiSearchFirstIndexCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 @@ -220,7 +222,6 @@ multiSearchFirstIndex(haystack, \[needle1, needle2, …, n ## multiSearchAny {#multisearchany} -Returns 1, if at least one string needlei matches the string `haystack` and 0 otherwise. 至少已有一个子字符串`needle`匹配 `haystack` 时返回1，否则返回 0 。 函数 `multiSearchAnyCaseInsensitive`, `multiSearchAnyUTF8` 和 `multiSearchAnyCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 @@ -237,7 +238,6 @@ multiSearchAny(haystack, [needle1, needle2, …, needleN]) 返回字符串 `haystack` 是否匹配正则表达式 `pattern` （[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) 匹配基于 UTF-8，例如`.` 匹配 Unicode 代码点 `¥`，它使用两个字节以 UTF-8 表示。T正则表达式不得包含空字节。如果 `haystack` 或`pattern`不是有效的 UTF-8，则此行为为undefined。 - 与 re2 的默认行为不同，`.` 会匹配换行符。要禁用此功能，请在模式前面添加`(?-s)`。 如果仅希望搜索子字符串，可以使用函数 [like](#like)或 [position](#position) 来替代，这些函数的性能比此函数更高。 @@ -447,8 +447,8 @@ SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[ 不会自动执行 Unicode 规范化，您可以使用[normalizeUTF8*()](https://clickhouse.com/docs/zh/sql-reference/functions/string-functions/) 函数来执行此操作。 -匹配字面上的 `%`, `_` 和 `/`（这些是 LIKE 元字符），请在其前面加上反斜杠：`\%`, `\_` 和 `\\`。 -如果在反斜杠前使用非 `%`, `_` 或 `\` 字符，则反斜杠将失去其特殊含义（即被解释为字面值）。 +如果需要匹配字符 `%`, `_` 和 `/`（这些是 LIKE 元字符），请在其前面加上反斜杠：`\%`, `\_` 和 `\\`。 +如果在非 `%`, `_` 或 `\` 字符前使用反斜杠，则反斜杠将失去其特殊含义（即被解释为字面值）。 请注意，ClickHouse 要求字符串中使用反斜杠 [也需要被转义](../syntax.md#string), 因此您实际上需要编写 `\\%`、`\\_` 和 `\\\\`。 @@ -483,7 +483,7 @@ like(haystack, pattern) ## ngramDistance -计算字符串`haystack` 和子字符串`needle`的4-gram距离。 为此，它计算两个 4-gram 多重集之间的对称差异，并通过它们的基数之和对其进行标准化。返回0-1之间的Float32。返回值越小，代表字符串越相似. Throws an exception if constant `needle` or `haystack` arguments are more than 32Kb in size. If any of non-constant `haystack` or `needle` arguments is more than 32Kb in size, the distance is always 1. +计算字符串 `haystack` 和子字符串 `needle` 的 4-gram 距离。 为此，它计算两个 4-gram 多重集之间的对称差异，并通过它们的基数之和对其进行标准化。返回 0 到 1 之间的 Float32 浮点数。返回值越小，代表字符串越相似. 如果参数 `needle` or `haystack` 是常数且大小超过 32Kb，则抛出异常。如果参数 `haystack` 或 `needle` 是非常数且大小超过 32Kb ，则返回值恒为 1。 函数 `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 @@ -495,11 +495,12 @@ ngramDistance(haystack, needle) ## ngramSearch -类似于`ngramDistance`，但计算`needle`字符串和`haystack`字符串之间的非对称差异，即来自needle的n-gram数量减去由`needle`数量归一化的n-gram的公共数量n-gram。返回 0 到 1 之间的 Float32。结果越大，’needle’越有可能在’haystack’中。该函数对于模糊字符串搜索很有用。另请参阅函数’soundex’。 +类似于`ngramDistance`，但计算`needle`字符串和 `haystack` 字符串之间的非对称差异，即来自 `needle` 的 n-gram 数量减去由`needle`数量归一化的 n-gram 的公共数量 n-gram。返回 0 到 1 之间的 Float32 浮点数。结果越大，`needle` 越有可能在 `haystack` 中。该函数对于模糊字符串搜索很有用。另请参阅函数 `soundex``。 + 函数 `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 :::note -UTF-8变体使用了3-gram距离。这些并不是完全公平的n-gram距离。我们使用2字节的哈希函数来哈希n-gram，然后计算这些哈希表之间的(非)对称差异——可能会发生冲突。在使用UTF-8大小写不敏感格式时，我们并不使用公平的tolower函数——我们将每个码点字节的第5位（从零开始）和如果字节超过一个的零字节的第一位置零——这对拉丁字母和大部分西里尔字母都有效。 +UTF-8 变体使用了 3-gram 距离。这些并不是完全公平的 n-gram 距离。我们使用 2 字节的哈希函数来哈希 n-gram，然后计算这些哈希表之间的(非)对称差异——可能会发生冲突。在使用 UTF-8 大小写不敏感格式时，我们并不使用公平的 `tolower` 函数——我们将每个码点字节的第 5 位（从零开始）和第零字节的第一个比特位位置为零（如果该串的大小超过一个字节）——这对拉丁字母和大部分西里尔字母都有效。 ::: **语法** @@ -512,7 +513,7 @@ ngramSearch(haystack, needle) 返回字符串 `haystack` 中子字符串 `needle` 出现的次数。 -函数 `countSubstringsCaseInsensitive` and `countSubstringsCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 +函数 `countSubstringsCaseInsensitive` 和 `countSubstringsCaseInsensitiveUTF8` 提供此函数的不区分大小写以及 UTF-8 变体。 **语法** @@ -522,9 +523,9 @@ countSubstrings(haystack, needle[, start_pos]) **参数** -- `haystack` — 被检索查询字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — 子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – 在字符串`haystack` 中开始检索的位置(从1开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选 +- `haystack` — 被搜索的字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 用于搜索的模式子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` – 在字符串`haystack` 中开始检索的位置(从 1 开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选。 **返回值** @@ -562,7 +563,7 @@ SELECT countSubstrings('abc___abc', 'abc', 4); ## countMatches -返回正则表达式成功匹配的次数。 +返回正则表达式 `pattern` 在 `haystack` 中成功匹配的次数。 **语法** @@ -573,7 +574,7 @@ countMatches(haystack, pattern) **参数** - `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). -- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含group，每个group用括号括起来。 如果 `pattern` 不包含group则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax)） 数据类型为[String](../../sql-reference/data-types/string.md). **返回值** @@ -627,11 +628,11 @@ regexpExtract(haystack, pattern[, index]) - `haystack` — 被匹配字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). - `pattern` — 正则表达式，必须是常量。 [String](../../sql-reference/syntax.md#syntax-string-literal). -- `index` – 一个大于等于0的整数，默认为1，它代表要提取哪个正则表达式组。 [UInt or Int](../../sql-reference/data-types/int-uint.md) 可选。 +- `index` – 一个大于等于 0 的整数，默认为 1 ，它代表要提取哪个正则表达式组。 [UInt or Int](../../sql-reference/data-types/int-uint.md) 可选。 **返回值** -`pattern`可以包含多个正则组, `index` 代表要提取哪个正则表达式组。如果 `index` 为0，则返回整个匹配的字符串。 +`pattern`可以包含多个正则组, `index` 代表要提取哪个正则表达式组。如果 `index` 为 0，则返回整个匹配的字符串。 数据类型: `String`. @@ -655,7 +656,7 @@ SELECT ## hasSubsequence -如果`needle`是`haystack`的子序列，返回1，否贼返回0。 +如果`needle`是`haystack`的子序列，返回1，否则返回0。 子序列是从给定字符串中删除零个或多个元素而不改变剩余元素的顺序得到的序列。 **语法** @@ -691,7 +692,7 @@ SELECT hasSubsequence('garbage', 'arg') ; ``` ## hasSubsequenceCaseInsensitive -类似于[hasSubsequence](#hasSubsequence)但是不区分大小写。 +类似于 [hasSubsequence](#hasSubsequence) 但是不区分大小写。 ## hasSubsequenceUTF8 From 51db06992d01917ec5d5aa4aa76489c0a2767c07 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Mon, 1 Apr 2024 17:54:48 +0200 Subject: [PATCH 0801/1165] refine --- .../sql-reference/functions/string-search-functions.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/zh/sql-reference/functions/string-search-functions.md b/docs/zh/sql-reference/functions/string-search-functions.md index 3d88704a297..79b33a4b0bc 100644 --- a/docs/zh/sql-reference/functions/string-search-functions.md +++ b/docs/zh/sql-reference/functions/string-search-functions.md @@ -112,10 +112,9 @@ SELECT ## locate -类似于 [position](#position) but with arguments `haystack` and `locate` switched. +类似于 [position](#position) 但交换了 `haystack` 和 `locate` 参数。 此函数的行为取决于 ClickHouse 版本： -- in versions < v24.3, `locate` was an alias of function `position` and accepted arguments `(haystack, needle[, start_pos])`. - 在 v24.3 以下的版本中，`locate` 是函数`position`的别名，参数为 `(haystack, needle[, start_pos])`。 - 在 v24.3 及以上的版本中,, `locate` 是独立的函数 (以更好地兼容 MySQL) ,参数为 `(needle, haystack[, start_pos])`。 之前的行为 可以在设置中恢复 [function_locate_has_mysql_compatible_argument_order = false](../../operations/settings/settings.md#function-locate-has-mysql-compatible-argument-order); @@ -200,7 +199,7 @@ SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']); 类似于 `position` , 在字符串`haystack`中匹配多个`needle`子字符串，从左开始任一匹配的子串，返回其位置。 -函数 `multiSearchFirstPositionCaseInsensitive`, `multiSearchFirstPositionUTF8` and `multiSearchFirstPositionCaseInsensitiveUTF8` 提供此函数的不区分大小写 以及/或 UTF-8 变体。 +函数 `multiSearchFirstPositionCaseInsensitive`, `multiSearchFirstPositionUTF8` 和 `multiSearchFirstPositionCaseInsensitiveUTF8` 提供此函数的不区分大小写 以及/或 UTF-8 变体。 **语法** @@ -374,7 +373,7 @@ extractAllGroupsHorizontal(haystack, pattern) **参数** - `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). -- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含group，每个group用括号括起来。 如果 `pattern` 不包含group则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含 group，每个 group 用括号括起来。 如果 `pattern` 不包含 group 则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). **返回值** @@ -627,7 +626,7 @@ regexpExtract(haystack, pattern[, index]) **参数** - `haystack` — 被匹配字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). -- `pattern` — 正则表达式，必须是常量。 [String](../../sql-reference/syntax.md#syntax-string-literal). +- `pattern` — 正则表达式，必须是常量。类型为[String](../../sql-reference/syntax.md#syntax-string-literal). - `index` – 一个大于等于 0 的整数，默认为 1 ，它代表要提取哪个正则表达式组。 [UInt or Int](../../sql-reference/data-types/int-uint.md) 可选。 **返回值** From 99a5e1ed2d114ddd6d9c6b3d8845a9265d20609e Mon Sep 17 00:00:00 2001 From: Han Fei Date: Mon, 1 Apr 2024 17:58:25 +0200 Subject: [PATCH 0802/1165] improve --- docs/zh/sql-reference/functions/string-search-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/sql-reference/functions/string-search-functions.md b/docs/zh/sql-reference/functions/string-search-functions.md index 79b33a4b0bc..972fd84e2a1 100644 --- a/docs/zh/sql-reference/functions/string-search-functions.md +++ b/docs/zh/sql-reference/functions/string-search-functions.md @@ -209,7 +209,7 @@ multiSearchFirstPosition(haystack, [needle1, needle2, …, needleN]) ## multiSearchFirstIndex -在字符串`haystack`中匹配最左侧的`needle`子字符串i，返回其索引 `i` (从1开始)，如无法匹配则返回0。 +在字符串`haystack`中匹配最左侧的 needlei 子字符串，返回其索引 `i` (从1开始)，如无法匹配则返回0。 函数 `multiSearchFirstIndexCaseInsensitive`, `multiSearchFirstIndexUTF8` 和 `multiSearchFirstIndexCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 From 2a5e9ea7136f9f2ec9a5334cd5b82799d3110715 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 1 Apr 2024 13:07:52 -0300 Subject: [PATCH 0803/1165] fix black --- tests/clickhouse-test | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 0661380e061..1f0c0a131e4 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -71,6 +71,7 @@ VERSION_PATTERN = r"^((\d+\.)?(\d+\.)?(\d+\.)?\d+)$" TEST_MAX_RUN_TIME_IN_SECONDS = 120 + class SharedEngineReplacer: ENGINES_NON_REPLICATED_REGEXP = r"[ =]((Collapsing|VersionedCollapsing|Summing|Replacing|Aggregating|)MergeTree\(?\)?)" ENGINES_MAPPING_REPLICATED = [ @@ -683,7 +684,9 @@ class FailureReason(enum.Enum): STDERR = "having stderror: " EXCEPTION = "having exception in stdout: " RESULT_DIFF = "result differs with reference: " - TOO_LONG = f"Test runs too long (> {TEST_MAX_RUN_TIME_IN_SECONDS}s). Make it faster." + TOO_LONG = ( + f"Test runs too long (> {TEST_MAX_RUN_TIME_IN_SECONDS}s). Make it faster." + ) INTERNAL_QUERY_FAIL = "Internal query (CREATE/DROP DATABASE) failed:" # SKIPPED reasons From 2a183fcbef32152534846bb569013e686284ef76 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 1 Apr 2024 16:29:39 +0000 Subject: [PATCH 0804/1165] fix build --- src/Databases/DatabasesCommon.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 824e9fd7b4d..c074bf201bd 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -348,7 +348,7 @@ StoragePtr DatabaseWithOwnTablesBase::getTableUnlocked(const String & table_name backQuote(database_name), backQuote(table_name)); } -std::vector> DatabaseWithOwnTablesBase::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool skip_not_loaded) const +std::vector> DatabaseWithOwnTablesBase::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const { std::vector> res; From aacb65299b21ab07cbf94a9d9b5d5a8c761b8935 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 1 Apr 2024 16:45:45 +0000 Subject: [PATCH 0805/1165] fix --- src/Databases/PostgreSQL/DatabasePostgreSQL.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index b07b203f786..3f62b9719d2 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -97,7 +97,7 @@ bool DatabasePostgreSQL::empty() const } -DatabaseTablesIteratorPtr DatabasePostgreSQL::getTablesIterator(ContextPtr local_context, const FilterByNameFunction & /* filter_by_table_name */) const +DatabaseTablesIteratorPtr DatabasePostgreSQL::getTablesIterator(ContextPtr local_context, const FilterByNameFunction & /* filter_by_table_name */, bool /* skip_not_loaded */) const { std::lock_guard lock(mutex); Tables tables; From 9c98f47f97c6a6c260a2969cd9ccfaafd22d0740 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 1 Apr 2024 16:53:49 +0000 Subject: [PATCH 0806/1165] more fixes --- src/Databases/DatabaseHDFS.cpp | 2 +- src/Databases/DatabaseOrdinary.cpp | 2 +- src/Databases/DatabaseReplicated.cpp | 2 +- src/Databases/DatabasesCommon.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp index 2688ff2443c..1de7f80f512 100644 --- a/src/Databases/DatabaseHDFS.cpp +++ b/src/Databases/DatabaseHDFS.cpp @@ -225,7 +225,7 @@ std::vector> DatabaseHDFS::getTablesForBackup(cons * Returns an empty iterator because the database does not have its own tables * But only caches them for quick access */ -DatabaseTablesIteratorPtr DatabaseHDFS::getTablesIterator(ContextPtr, const FilterByNameFunction &) const +DatabaseTablesIteratorPtr DatabaseHDFS::getTablesIterator(ContextPtr, const FilterByNameFunction &, bool) const { return std::make_unique(Tables{}, getDatabaseName()); } diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 3859c2fe0ce..e9168d68ef7 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -460,7 +460,7 @@ DatabaseTablesIteratorPtr DatabaseOrdinary::getTablesIterator(ContextPtr local_c return DatabaseWithOwnTablesBase::getTablesIterator(local_context, filter_by_table_name, skip_not_loaded); } -Strings DatabaseOrdinary::getAllTableNames(ContextPtr context) const +Strings DatabaseOrdinary::getAllTableNames(ContextPtr) const { std::set unique_names; { diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 3b1646f1903..7b8f7468e81 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -873,7 +873,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep std::vector replicated_tables_to_rename; size_t total_tables = 0; std::vector replicated_ids; - for (auto existing_tables_it = getTablesIterator(getContext(), {}); existing_tables_it->isValid(); + for (auto existing_tables_it = getTablesIterator(getContext(), {}, /*skip_not_loaded=*/false); existing_tables_it->isValid(); existing_tables_it->next(), ++total_tables) { String name = existing_tables_it->name(); diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index c074bf201bd..57385c98c15 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -352,7 +352,7 @@ std::vector> DatabaseWithOwnTablesBase::getTablesF { std::vector> res; - for (auto it = getTablesIterator(local_context, filter); it->isValid(); it->next()) + for (auto it = getTablesIterator(local_context, filter, /*skip_not_loaded=*/false); it->isValid(); it->next()) { auto storage = it->table(); if (!storage) From f0642485d7191d7aeb9d0716382efc4d4f5ff9d2 Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 1 Apr 2024 19:31:02 +0200 Subject: [PATCH 0807/1165] Add examples for UTF8 variants of ngramDistance --- .../functions/string-search-functions.md | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index e93971879a2..dda1d2170dd 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -580,13 +580,27 @@ ngramDistanceUTF8(haystack, needle) **Parameters** -- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) **Returned value** - Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) +**Example** + +Query: + +```sql +SELECT ngramDistanceUTF8('abcde','cde'); +``` + +Result: + +```response +0.5 +``` + ## ngramDistanceCaseInsensitiveUTF8 Provides a case-insensitive variant of [ngramDistanceUTF8](#ngramdistanceutf8). @@ -599,13 +613,26 @@ ngramDistanceCaseInsensitiveUTF8(haystack, needle) **Parameters** -- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) **Returned value** - Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) +**Example** + +Query: + +```sql +SELECT ngramDistanceCaseInsensitiveUTF8('abcde','CDE'); +``` + +Result: + +```response +0.5 +``` ## ngramSearch @@ -628,6 +655,7 @@ ngramSearch(haystack, needle) - Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + **Implementation details** :::note From 5e664b0f9dd28f4196cb56b275260b068d4511ad Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 1 Apr 2024 19:41:18 +0200 Subject: [PATCH 0808/1165] Correct link to string literal --- .../functions/string-search-functions.md | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index dda1d2170dd..dcfb34cde37 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -477,8 +477,8 @@ ngramDistance(haystack, needle) **Parameters** -- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) **Returned value** @@ -531,8 +531,8 @@ ngramDistanceCaseInsensitive(haystack, needle) **Parameters** -- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) **Returned value** @@ -580,8 +580,8 @@ ngramDistanceUTF8(haystack, needle) **Parameters** -- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) **Returned value** @@ -613,8 +613,8 @@ ngramDistanceCaseInsensitiveUTF8(haystack, needle) **Parameters** -- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) **Returned value** @@ -648,8 +648,8 @@ ngramSearch(haystack, needle) **Parameters** -- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) **Returned value** @@ -688,8 +688,8 @@ ngramSearchCaseInsensitive(haystack, needle) **Parameters** -- `haystack`: First comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) **Returned value** @@ -723,8 +723,8 @@ ngramSearchUTF8(haystack, needle) **Parameters** -- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) **Returned value** @@ -758,8 +758,8 @@ ngramSearchCaseInsensitiveUTF8(haystack, needle) **Parameters** -- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) -- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#syntax-string-literal) +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) **Returned value** From 3207f230f8e6769ccec8c5283a3fcfede98c18a5 Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 1 Apr 2024 20:00:30 +0200 Subject: [PATCH 0809/1165] Small fixes --- docs/en/sql-reference/functions/string-search-functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index dcfb34cde37..af4fc8edcd3 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -540,7 +540,7 @@ ngramDistanceCaseInsensitive(haystack, needle) **Examples** -With [ngramDistance](#ngramdistance) differences in case will drive up the similarity score: +With [ngramDistance](#ngramdistance) differences in case will affect the similarity value: Query: @@ -554,7 +554,7 @@ Result: 0.71428573 ``` -With [ngramDistanceCaseInsensitive](#ngramdistancecaseinsensitive) case is ignored so two identical strings differing only in case will now read as identical: +With [ngramDistanceCaseInsensitive](#ngramdistancecaseinsensitive) case is ignored so two identical strings differing only in case will now return a low similarity value: Query: From 4415dd86dff1f52a5a8921f3033b444f48549260 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 2 Apr 2024 09:33:48 +0800 Subject: [PATCH 0810/1165] remove code of checkpoint --- src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp index 6ddba781a54..1399217d977 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp @@ -45,7 +45,6 @@ HiveTextFormatReader::HiveTextFormatReader(PeekableReadBuffer & buf_, const Form std::vector HiveTextFormatReader::readNames() { - PeekableReadBufferCheckpoint checkpoint{*buf, true}; return input_field_names; } From 2f5c52f7dff0e3a5c9bd82d565dfc368fa2ab4f0 Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Tue, 2 Apr 2024 05:14:12 +0000 Subject: [PATCH 0811/1165] fix 0320_long_values_pretty_are_not_cut_if_single Signed-off-by: Duc Canh Le --- .../03020_long_values_pretty_are_not_cut_if_single.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/03020_long_values_pretty_are_not_cut_if_single.sh b/tests/queries/0_stateless/03020_long_values_pretty_are_not_cut_if_single.sh index fa9e9f6d3e1..b66951d93f6 100755 --- a/tests/queries/0_stateless/03020_long_values_pretty_are_not_cut_if_single.sh +++ b/tests/queries/0_stateless/03020_long_values_pretty_are_not_cut_if_single.sh @@ -9,6 +9,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # But cutting it in the result of SHOW CREATE TABLE will be bad for a user. # That's why we control it with the setting `output_format_pretty_max_value_width_apply_for_single_value`. +# Make sure that system.metric_log exists +${CLICKHOUSE_CLIENT} --query "SELECT 1 FORMAT Null" +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" + + ${CLICKHOUSE_CLIENT} --query "SHOW CREATE TABLE system.metric_log" --format Pretty | grep -P '^COMMENT' ${CLICKHOUSE_CLIENT} --query "SHOW CREATE TABLE system.metric_log" --format PrettyCompact | grep -P '^COMMENT' ${CLICKHOUSE_CLIENT} --query "SHOW CREATE TABLE system.metric_log" --format PrettySpace | grep -P '^COMMENT' From 9f51e9825d875ebb27b64467c879486afa8cb7bf Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 2 Apr 2024 11:02:23 +0200 Subject: [PATCH 0812/1165] Update NuRaft --- contrib/NuRaft | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index 4a12f99dfc9..08ac76ea80a 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 4a12f99dfc9d47c687ff7700b927cc76856225d1 +Subproject commit 08ac76ea80a37f89b12109c805eafe9f1dc9b991 From 24b02dc9b81df32f13256549abc9869151617e46 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 2 Apr 2024 11:10:09 +0200 Subject: [PATCH 0813/1165] Update CMake --- contrib/nuraft-cmake/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/nuraft-cmake/CMakeLists.txt b/contrib/nuraft-cmake/CMakeLists.txt index eaca00566d6..970ca4b9ce1 100644 --- a/contrib/nuraft-cmake/CMakeLists.txt +++ b/contrib/nuraft-cmake/CMakeLists.txt @@ -32,6 +32,7 @@ set(SRCS "${LIBRARY_DIR}/src/handle_custom_notification.cxx" "${LIBRARY_DIR}/src/handle_vote.cxx" "${LIBRARY_DIR}/src/launcher.cxx" + "${LIBRARY_DIR}/src/log_entry.cxx" "${LIBRARY_DIR}/src/srv_config.cxx" "${LIBRARY_DIR}/src/snapshot_sync_req.cxx" "${LIBRARY_DIR}/src/snapshot_sync_ctx.cxx" From 48ab0721a88611957fe42bc6c9e10975f8b7cbbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 2 Apr 2024 11:52:14 +0200 Subject: [PATCH 0814/1165] Unify lightweight mutation control --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Planner/PlannerJoinTree.cpp | 6 ++---- src/Storages/HDFS/StorageHDFS.h | 2 +- src/Storages/HDFS/StorageHDFSCluster.h | 2 +- src/Storages/IStorage.h | 5 ++++- src/Storages/MergeTree/MergeTreeData.cpp | 5 +++++ src/Storages/MergeTree/MergeTreeData.h | 2 +- src/Storages/RocksDB/StorageEmbeddedRocksDB.h | 2 +- src/Storages/StorageAzureBlob.h | 2 +- src/Storages/StorageAzureBlobCluster.h | 2 +- src/Storages/StorageFile.h | 2 +- src/Storages/StorageFileCluster.h | 2 +- src/Storages/StorageJoin.h | 2 +- src/Storages/StorageMaterializedMySQL.h | 2 +- src/Storages/StorageMerge.cpp | 4 ++-- src/Storages/StorageMerge.h | 2 +- src/Storages/StorageS3.h | 2 +- src/Storages/StorageS3Cluster.h | 2 +- src/Storages/StorageURL.h | 2 +- src/Storages/StorageURLCluster.h | 2 +- 20 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6bbf03bb1e0..dee522a2184 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2298,7 +2298,7 @@ std::optional InterpreterSelectQuery::getTrivialCount(UInt64 max_paralle && !settings.allow_experimental_query_deduplication && !settings.empty_result_for_aggregation_by_empty_set && storage - && storage->supportsTrivialCountOptimization() + && storage->supportsTrivialCountOptimization(storage_snapshot, getContext()) && query_info.filter_asts.empty() && query_analyzer->hasAggregation() && (query_analyzer->aggregates().size() == 1) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index dddab524101..d2f37ff1ad4 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -225,7 +225,8 @@ bool applyTrivialCountIfPossible( return false; const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); - if (!storage->supportsTrivialCountOptimization()) + if (!storage->supportsTrivialCountOptimization( + table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot(), query_context)) return false; auto storage_id = storage->getStorageID(); @@ -262,9 +263,6 @@ bool applyTrivialCountIfPossible( if (main_query_node.hasGroupBy() || main_query_node.hasPrewhere() || main_query_node.hasWhere()) return false; - if (storage->hasLightweightDeletedMask()) - return false; - if (settings.allow_experimental_query_deduplication || settings.empty_result_for_aggregation_by_empty_set) return false; diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index b14bb7f997b..b8faa27d678 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -92,7 +92,7 @@ public: static SchemaCache & getSchemaCache(const ContextPtr & ctx); - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } protected: friend class HDFSSource; diff --git a/src/Storages/HDFS/StorageHDFSCluster.h b/src/Storages/HDFS/StorageHDFSCluster.h index 26ebc8601ee..0b5c6242aa9 100644 --- a/src/Storages/HDFS/StorageHDFSCluster.h +++ b/src/Storages/HDFS/StorageHDFSCluster.h @@ -36,7 +36,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 1108eafc6b6..87a04c3fcc6 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -265,7 +265,10 @@ public: /// Return true if the trivial count query could be optimized without reading the data at all /// in totalRows() or totalRowsByPartitionPredicate() methods or with optimized reading in read() method. - virtual bool supportsTrivialCountOptimization() const { return false; } + virtual bool supportsTrivialCountOptimization(const StorageSnapshotPtr & /*storage_snapshot*/, ContextPtr /*query_context*/) const + { + return false; + } private: StorageID storage_id; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e984f306e2e..8faed72b198 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -8221,6 +8221,11 @@ void MergeTreeData::updateObjectColumns(const DataPartPtr & part, const DataPart DB::updateObjectColumns(object_columns, columns, part->getColumns()); } +bool MergeTreeData::supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const +{ + return !hasLightweightDeletedMask(); +} + StorageSnapshotPtr MergeTreeData::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { auto snapshot_data = std::make_unique(); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 046376be474..0d56b902f1a 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -440,7 +440,7 @@ public: bool areAsynchronousInsertsEnabled() const override { return getSettings()->async_insert; } - bool supportsTrivialCountOptimization() const override { return !hasLightweightDeletedMask(); } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override; /// Snapshot for MergeTree contains the current set of data parts /// at the moment of the start of query. diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.h b/src/Storages/RocksDB/StorageEmbeddedRocksDB.h index 8525108735b..230464a161f 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.h +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.h @@ -93,7 +93,7 @@ public: bool supportsDelete() const override { return true; } /// To turn on the optimization optimize_trivial_approximate_count_query=1 should be set for a query. - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } std::optional totalRows(const Settings & settings) const override; diff --git a/src/Storages/StorageAzureBlob.h b/src/Storages/StorageAzureBlob.h index 27ac7a5c368..3f1ba33f636 100644 --- a/src/Storages/StorageAzureBlob.h +++ b/src/Storages/StorageAzureBlob.h @@ -100,7 +100,7 @@ public: bool supportsSubsetOfColumns(const ContextPtr & context) const; - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } bool prefersLargeBlocks() const override; diff --git a/src/Storages/StorageAzureBlobCluster.h b/src/Storages/StorageAzureBlobCluster.h index 545e568a772..eff4d70f1bd 100644 --- a/src/Storages/StorageAzureBlobCluster.h +++ b/src/Storages/StorageAzureBlobCluster.h @@ -35,7 +35,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: void updateBeforeRead(const ContextPtr & /*context*/) override {} diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 93c263008a6..588429284f0 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -134,7 +134,7 @@ public: const ContextPtr & context, size_t & total_bytes_to_read); - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } protected: friend class StorageFileSource; diff --git a/src/Storages/StorageFileCluster.h b/src/Storages/StorageFileCluster.h index 3acbc71ba7e..973d595bbf0 100644 --- a/src/Storages/StorageFileCluster.h +++ b/src/Storages/StorageFileCluster.h @@ -32,7 +32,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index dc68c68a21b..c76df0cb452 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -85,7 +85,7 @@ public: const Names & getKeyNames() const { return key_names; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: Block sample_block; diff --git a/src/Storages/StorageMaterializedMySQL.h b/src/Storages/StorageMaterializedMySQL.h index 9f5d157ce3b..3e0cf3e6925 100644 --- a/src/Storages/StorageMaterializedMySQL.h +++ b/src/Storages/StorageMaterializedMySQL.h @@ -40,7 +40,7 @@ public: void drop() override { nested_storage->drop(); } - bool supportsTrivialCountOptimization() const override { return false; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return false; } IndexSizeByName getSecondaryIndexSizes() const override { diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 1b5e4860464..5eceddfe06d 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1675,9 +1675,9 @@ std::tuple StorageMerge::evaluateDatabaseName(cons return {false, ast}; } -bool StorageMerge::supportsTrivialCountOptimization() const +bool StorageMerge::supportsTrivialCountOptimization(const StorageSnapshotPtr & storage_snapshot, ContextPtr ctx) const { - return getFirstTable([&](const auto & table) { return !table->supportsTrivialCountOptimization(); }) == nullptr; + return getFirstTable([&](const auto & table) { return !table->supportsTrivialCountOptimization(storage_snapshot, ctx); }) == nullptr; } std::optional StorageMerge::totalRows(const Settings & settings) const diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index c049d50f3b4..a63ea1e32ef 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -76,7 +76,7 @@ public: /// Evaluate database name or regexp for StorageMerge and TableFunction merge static std::tuple evaluateDatabaseName(const ASTPtr & node, ContextPtr context); - bool supportsTrivialCountOptimization() const override; + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override; std::optional totalRows(const Settings & settings) const override; std::optional totalBytes(const Settings & settings) const override; diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index d1f15edfd6d..19cbfaa6f08 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -352,7 +352,7 @@ public: using KeysWithInfo = StorageS3Source::KeysWithInfo; - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } protected: virtual Configuration updateConfigurationAndGetCopy(const ContextPtr & local_context); diff --git a/src/Storages/StorageS3Cluster.h b/src/Storages/StorageS3Cluster.h index 6a5b03e682f..802fd3f9139 100644 --- a/src/Storages/StorageS3Cluster.h +++ b/src/Storages/StorageS3Cluster.h @@ -32,7 +32,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } protected: void updateConfigurationIfChanged(ContextPtr local_context); diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 842cfd5b627..5aca3df1513 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -127,7 +127,7 @@ protected: bool parallelizeOutputAfterReading(ContextPtr context) const override; - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: static std::pair getTableStructureAndFormatFromDataImpl( diff --git a/src/Storages/StorageURLCluster.h b/src/Storages/StorageURLCluster.h index dce2e0106ea..c80cdec74a2 100644 --- a/src/Storages/StorageURLCluster.h +++ b/src/Storages/StorageURLCluster.h @@ -35,7 +35,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; From 70e8477a4815d8b96772a911e45488456659db3b Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 2 Apr 2024 17:53:23 +0800 Subject: [PATCH 0815/1165] add no-fasttest tag --- .../queries/0_stateless/03033_hive_text_read_variable_fields.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03033_hive_text_read_variable_fields.sh b/tests/queries/0_stateless/03033_hive_text_read_variable_fields.sh index 5af0e465cca..9dba99be7c8 100755 --- a/tests/queries/0_stateless/03033_hive_text_read_variable_fields.sh +++ b/tests/queries/0_stateless/03033_hive_text_read_variable_fields.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# Tags: no-fasttest # NOTE: this sh wrapper is required because of shell_config CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) From fb94a954258ebaeb6ca3fcecb6164243e4eb8d33 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 2 Apr 2024 12:23:22 +0200 Subject: [PATCH 0816/1165] Add logging --- src/Interpreters/Cache/IFileCachePriority.h | 4 ++-- src/Interpreters/Cache/LRUFileCachePriority.cpp | 13 +++++++++++-- src/Interpreters/Cache/LRUFileCachePriority.h | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index ff06f17ce36..8dcc114d9cd 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -192,8 +192,8 @@ protected: virtual void releaseImpl(size_t /* size */, size_t /* elements */) {} - size_t max_size = 0; - size_t max_elements = 0; + std::atomic max_size = 0; + std::atomic max_elements = 0; }; } diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index e65c102f1e3..1d9725352be 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -439,12 +439,15 @@ void LRUFileCachePriority::LRUIterator::invalidate() assertValid(); const auto & entry = *iterator; - LOG_TEST(cache_priority->log, - "Invalidating entry in LRU queue entry {}", entry->toString()); chassert(entry->size != 0); cache_priority->updateSize(-entry->size); cache_priority->updateElementsCount(-1); + + LOG_TEST(cache_priority->log, + "Invalidated entry in LRU queue {}: {}", + entry->toString(), cache_priority->getApproxStateInfoForLog()); + entry->size = 0; } @@ -521,6 +524,12 @@ std::string LRUFileCachePriority::getStateInfoForLog(const CachePriorityGuard::L getSize(lock), max_size, getElementsCount(lock), max_elements, description); } +std::string LRUFileCachePriority::getApproxStateInfoForLog() const +{ + return fmt::format("size: {}/{}, elements: {}/{} (description: {})", + getSizeApprox(), max_size, getElementsCountApprox(), max_elements, description); +} + void LRUFileCachePriority::holdImpl( size_t size, size_t elements, diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index 31968d61196..6627fcf1dee 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -116,6 +116,7 @@ private: const CachePriorityGuard::Lock & lock) override; void releaseImpl(size_t size, size_t elements) override; + std::string getApproxStateInfoForLog() const; }; class LRUFileCachePriority::LRUIterator : public IFileCachePriority::Iterator From 0d6230717898adc62ef8872f39da080b4d231c8c Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 2 Apr 2024 10:34:52 +0000 Subject: [PATCH 0817/1165] Update tests --- tests/queries/0_stateless/01601_accurate_cast.reference | 1 - tests/queries/0_stateless/01601_accurate_cast.sql | 2 +- .../0_stateless/02303_cast_nullable_to_custom_types.reference | 4 ---- .../0_stateless/02303_cast_nullable_to_custom_types.sql | 4 ++-- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/01601_accurate_cast.reference b/tests/queries/0_stateless/01601_accurate_cast.reference index dbf9666f4cd..82138e6354a 100644 --- a/tests/queries/0_stateless/01601_accurate_cast.reference +++ b/tests/queries/0_stateless/01601_accurate_cast.reference @@ -10,7 +10,6 @@ 1970-01-01 00:00:19 2023-05-30 1970-01-20 -\N true false true diff --git a/tests/queries/0_stateless/01601_accurate_cast.sql b/tests/queries/0_stateless/01601_accurate_cast.sql index d2ecede2402..471e4e34a4a 100644 --- a/tests/queries/0_stateless/01601_accurate_cast.sql +++ b/tests/queries/0_stateless/01601_accurate_cast.sql @@ -35,7 +35,7 @@ SELECT accurateCast('1xxx', 'Date'); -- { serverError CANNOT_PARSE_DATE } SELECT accurateCast('2023-05-30', 'Date'); SELECT accurateCast(19, 'Date'); -select accurateCast('test', 'Nullable(Bool)'); +select accurateCast('test', 'Nullable(Bool)'); -- { serverError CANNOT_PARSE_BOOL } select accurateCast('test', 'Bool'); -- { serverError CANNOT_PARSE_BOOL } select accurateCast('truex', 'Bool'); -- { serverError CANNOT_PARSE_BOOL } select accurateCast('xfalse', 'Bool'); -- { serverError CANNOT_PARSE_BOOL } diff --git a/tests/queries/0_stateless/02303_cast_nullable_to_custom_types.reference b/tests/queries/0_stateless/02303_cast_nullable_to_custom_types.reference index 717484d4670..a257755481e 100644 --- a/tests/queries/0_stateless/02303_cast_nullable_to_custom_types.reference +++ b/tests/queries/0_stateless/02303_cast_nullable_to_custom_types.reference @@ -39,7 +39,3 @@ fuzzer issue \N \N \N -\N -\N -\N -\N diff --git a/tests/queries/0_stateless/02303_cast_nullable_to_custom_types.sql b/tests/queries/0_stateless/02303_cast_nullable_to_custom_types.sql index b56ebc2b09d..570fbcde01f 100644 --- a/tests/queries/0_stateless/02303_cast_nullable_to_custom_types.sql +++ b/tests/queries/0_stateless/02303_cast_nullable_to_custom_types.sql @@ -24,7 +24,7 @@ select toIPv6OrNull(number % 2 ? '' : NULL) from numbers(2); select IPv6StringToNum(number % 2 ? '0000:0000:0000:0000:0000:0000:0000:0000' : NULL) from numbers(2); select 'fuzzer issue'; -SELECT CAST(if(number % 2, 'truetrue', NULL), 'Nullable(Bool)') FROM numbers(2); -SELECT CAST(if(number % 2, 'falsefalse', NULL), 'Nullable(Bool)') FROM numbers(2); +SELECT CAST(if(number % 2, 'truetrue', NULL), 'Nullable(Bool)') FROM numbers(2); -- {serverError CANNOT_PARSE_BOOL} +SELECT CAST(if(number % 2, 'falsefalse', NULL), 'Nullable(Bool)') FROM numbers(2); -- {serverError CANNOT_PARSE_BOOL} SELECT accurateCastOrNull(if(number % 2, NULL, 'truex'), 'Bool') FROM numbers(4); SELECT accurateCastOrNull(if(number % 2, 'truex', NULL), 'Bool') FROM numbers(4); From 3914a0f67a1754b9ed8ef3ee99c0edb0c75c48d7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 2 Apr 2024 11:01:02 +0000 Subject: [PATCH 0818/1165] Document SYSTEM RELOAD ASYNCHRONOUS METRICS --- docs/en/sql-reference/statements/system.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 1dee2eac698..991a272166a 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -64,6 +64,14 @@ RELOAD FUNCTIONS [ON CLUSTER cluster_name] RELOAD FUNCTION [ON CLUSTER cluster_name] function_name ``` +## RELOAD ASYNCHRONOUS METRICS + +Re-calculates all [asynchronous metrics](../../operations/system-tables/asynchronous_metrics.md). Since asynchronous metrics are periodically updated based on setting [asynchronous_metrics_update_period_s](../../operations/server-configuration-parameters/settings.md), updating them manually using this statment is not necessary. + +```sql +RELOAD ASYNCHRONOUS METRICS [ON CLUSTER cluster_name] +``` + ## DROP DNS CACHE Clears ClickHouse’s internal DNS cache. Sometimes (for old ClickHouse versions) it is necessary to use this command when changing the infrastructure (changing the IP address of another ClickHouse server or the server used by dictionaries). From 13c348c3c8e95a5c4aed3c28e70e365cbd4e16eb Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 2 Apr 2024 13:24:11 +0200 Subject: [PATCH 0819/1165] Fix spelling --- docs/en/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 991a272166a..b35e9426297 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -66,7 +66,7 @@ RELOAD FUNCTION [ON CLUSTER cluster_name] function_name ## RELOAD ASYNCHRONOUS METRICS -Re-calculates all [asynchronous metrics](../../operations/system-tables/asynchronous_metrics.md). Since asynchronous metrics are periodically updated based on setting [asynchronous_metrics_update_period_s](../../operations/server-configuration-parameters/settings.md), updating them manually using this statment is not necessary. +Re-calculates all [asynchronous metrics](../../operations/system-tables/asynchronous_metrics.md). Since asynchronous metrics are periodically updated based on setting [asynchronous_metrics_update_period_s](../../operations/server-configuration-parameters/settings.md), updating them manually using this statement is typically not necessary. ```sql RELOAD ASYNCHRONOUS METRICS [ON CLUSTER cluster_name] From a8f3a07f1ff81dfb253919f57b8cf801f13783c8 Mon Sep 17 00:00:00 2001 From: yariks5s Date: Tue, 2 Apr 2024 11:35:41 +0000 Subject: [PATCH 0820/1165] init --- docs/en/operations/system-tables/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index eaf79d035a9..d9800e05ff9 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -47,7 +47,7 @@ An example: ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024 --> 7500 - 1048576 + 1048576 8192 524288 false From 6018434f8246ea7598c6af4dc4d59b0fdf6bf630 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 2 Apr 2024 19:37:23 +0800 Subject: [PATCH 0821/1165] add config input_format_hive_text_allow_variable_number_of_columns --- src/Core/Settings.h | 1 + src/Formats/FormatFactory.cpp | 1 + src/Formats/FormatSettings.h | 1 + src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 84e709294aa..e66a56e6cea 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -1009,6 +1009,7 @@ class IColumn; M(Char, input_format_hive_text_fields_delimiter, '\x01', "Delimiter between fields in Hive Text File", 0) \ M(Char, input_format_hive_text_collection_items_delimiter, '\x02', "Delimiter between collection(array or map) items in Hive Text File", 0) \ M(Char, input_format_hive_text_map_keys_delimiter, '\x03', "Delimiter between a pair of map key/values in Hive Text File", 0) \ + M(Bool, input_format_hive_text_allow_variable_number_of_columns, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values", 0) \ M(UInt64, input_format_msgpack_number_of_columns, 0, "The number of columns in inserted MsgPack data. Used for automatic schema inference from data.", 0) \ M(MsgPackUUIDRepresentation, output_format_msgpack_uuid_representation, FormatSettings::MsgPackUUIDRepresentation::EXT, "The way how to output UUID in MsgPack format.", 0) \ M(UInt64, input_format_max_rows_to_read_for_schema_inference, 25000, "The maximum rows of data to read for automatic schema inference", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 8cbb1b9e563..bd41dc12fa7 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -96,6 +96,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.hive_text.fields_delimiter = settings.input_format_hive_text_fields_delimiter; format_settings.hive_text.collection_items_delimiter = settings.input_format_hive_text_collection_items_delimiter; format_settings.hive_text.map_keys_delimiter = settings.input_format_hive_text_map_keys_delimiter; + format_settings.hive_text.allow_variable_number_of_columns = settings.input_format_hive_text_allow_variable_number_of_columns; format_settings.custom.escaping_rule = settings.format_custom_escaping_rule; format_settings.custom.field_delimiter = settings.format_custom_field_delimiter; format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 5b7995e0da2..a239941469f 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -176,6 +176,7 @@ struct FormatSettings char fields_delimiter = '\x01'; char collection_items_delimiter = '\x02'; char map_keys_delimiter = '\x03'; + bool allow_variable_number_of_columns = true; Names input_field_names; } hive_text{}; diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp index 1399217d977..b64318e4093 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp @@ -19,7 +19,7 @@ static FormatSettings updateFormatSettings(const FormatSettings & settings, cons updated.date_time_input_format = FormatSettings::DateTimeInputFormat::BestEffort; updated.defaults_for_omitted_fields = true; updated.csv.delimiter = updated.hive_text.fields_delimiter; - updated.csv.allow_variable_number_of_columns = true; + updated.csv.allow_variable_number_of_columns = settings.hive_text.allow_variable_number_of_columns; if (settings.hive_text.input_field_names.empty()) updated.hive_text.input_field_names = header.getNames(); return updated; From e659071b789512fcfde108e73ea291d3a8fe86ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Tue, 2 Apr 2024 11:37:48 +0000 Subject: [PATCH 0822/1165] Improve docs --- docs/en/sql-reference/statements/alter/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md index dc6668c7983..7961315c193 100644 --- a/docs/en/sql-reference/statements/alter/index.md +++ b/docs/en/sql-reference/statements/alter/index.md @@ -56,7 +56,9 @@ Entries for finished mutations are not deleted right away (the number of preserv For non-replicated tables, all `ALTER` queries are performed synchronously. For replicated tables, the query just adds instructions for the appropriate actions to `ZooKeeper`, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas. -For all `ALTER` queries, you can use the [alter_sync](/docs/en/operations/settings/settings.md/#alter-sync) setting to set up waiting. +For `ALTER` queries that creates mutations (e.g.: including, but not limited to `UPDATE`, `DELETE`, `MATERIALIZE INDEX`, `MATERIALIZE PROJECTION`, `MATERIALIZE COLUMN`, `APPLY DELETED MASK`, `CLEAR STATISTIC`, `MATERIALIZE STATISTIC`) the synchronicity is defined by the [mutations_sync](/docs/en/operations/settings/settings.md/#mutations_sync) setting. + +For other `ALTER` queries which only modify the metadata, you can use the [alter_sync](/docs/en/operations/settings/settings.md/#alter-sync) setting to set up waiting. You can specify how long (in seconds) to wait for inactive replicas to execute all `ALTER` queries with the [replication_wait_for_inactive_replica_timeout](/docs/en/operations/settings/settings.md/#replication-wait-for-inactive-replica-timeout) setting. @@ -64,8 +66,6 @@ You can specify how long (in seconds) to wait for inactive replicas to execute a For all `ALTER` queries, if `alter_sync = 2` and some replicas are not active for more than the time, specified in the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. ::: -For `ALTER TABLE ... UPDATE|DELETE|MATERIALIZE INDEX|MATERIALIZE PROJECTION|MATERIALIZE COLUMN` queries the synchronicity is defined by the [mutations_sync](/docs/en/operations/settings/settings.md/#mutations_sync) setting. - ## Related content - Blog: [Handling Updates and Deletes in ClickHouse](https://clickhouse.com/blog/handling-updates-and-deletes-in-clickhouse) From fd58e4d08bba0110335065a018a32d08654c151b Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Tue, 2 Apr 2024 13:41:56 +0200 Subject: [PATCH 0823/1165] Better String to Variant(String) conversion --- src/Functions/FunctionsConversion.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 448a5fd8fc6..60c069f632c 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -4119,10 +4119,11 @@ private: }; } - if (isStringOrFixedString(removeNullable(removeLowCardinality(from_type)))) + auto variant_discr_opt = to_variant.tryGetVariantDiscriminator(*removeNullableOrLowCardinalityNullable(from_type)); + /// Cast String to Variant through parsing if it's not Variant(String). + if (isStringOrFixedString(removeNullable(removeLowCardinality(from_type))) && (!variant_discr_opt || to_variant.getVariants().size() > 1)) return createStringToVariantWrapper(); - auto variant_discr_opt = to_variant.tryGetVariantDiscriminator(*removeNullableOrLowCardinalityNullable(from_type)); if (!variant_discr_opt) throw Exception(ErrorCodes::CANNOT_CONVERT_TYPE, "Cannot convert type {} to {}. Conversion to Variant allowed only for types from this Variant", from_type->getName(), to_variant.getName()); From 40a22ffa63453f3ee4db7e17e5d4d0d4023a886d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 2 Apr 2024 13:43:14 +0200 Subject: [PATCH 0824/1165] Remove a few nested include dependencies --- src/Access/AccessRights.cpp | 4 +++- src/Common/AtomicLogger.h | 4 ++++ src/Common/CurrentThreadHelpers.cpp | 16 ++++++++++++++++ src/Common/CurrentThreadHelpers.h | 9 +++++++++ src/Common/Exception.cpp | 19 ++++++++++--------- src/Common/Exception.h | 16 ++++++++-------- src/Common/Jemalloc.cpp | 1 + src/Common/Logger.h | 13 +++++++++---- src/Common/ProfileEventsScope.cpp | 1 + src/Common/ProfileEventsScope.h | 3 ++- src/Common/logger_useful.h | 15 ++++++--------- src/Core/BackgroundSchedulePool.h | 21 ++++++++++----------- src/IO/TimeoutSetter.cpp | 2 +- src/Interpreters/ExternalLoader.cpp | 21 +++++++++++---------- src/Server/CertificateReloader.cpp | 3 ++- src/Storages/NATS/NATSHandler.cpp | 1 + 16 files changed, 94 insertions(+), 55 deletions(-) create mode 100644 src/Common/CurrentThreadHelpers.cpp create mode 100644 src/Common/CurrentThreadHelpers.h diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 36a68bc0a34..a87e9361e8e 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -1,6 +1,8 @@ #include -#include #include +#include +#include + #include #include #include diff --git a/src/Common/AtomicLogger.h b/src/Common/AtomicLogger.h index 4bda55e070b..9581358218c 100644 --- a/src/Common/AtomicLogger.h +++ b/src/Common/AtomicLogger.h @@ -6,6 +6,8 @@ #include #include +namespace DB +{ /** AtomicLogger allows to atomically change logger. * Standard library does not have atomic_shared_ptr, and we do not use std::atomic* operations, @@ -49,3 +51,5 @@ private: mutable DB::SharedMutex log_mutex; LoggerPtr logger; }; + +} diff --git a/src/Common/CurrentThreadHelpers.cpp b/src/Common/CurrentThreadHelpers.cpp new file mode 100644 index 00000000000..cbfb50bf3b1 --- /dev/null +++ b/src/Common/CurrentThreadHelpers.cpp @@ -0,0 +1,16 @@ +#include +#include + +namespace DB +{ + +bool currentThreadHasGroup() +{ + return DB::CurrentThread::getGroup() != nullptr; +} + +LogsLevel currentThreadLogsLevel() +{ + return DB::CurrentThread::get().getClientLogsLevel(); +} +} diff --git a/src/Common/CurrentThreadHelpers.h b/src/Common/CurrentThreadHelpers.h new file mode 100644 index 00000000000..01a180e74d2 --- /dev/null +++ b/src/Common/CurrentThreadHelpers.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace DB +{ +bool currentThreadHasGroup(); +LogsLevel currentThreadLogsLevel(); +} diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 7e73e2c0783..7d38fdafddb 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -1,26 +1,27 @@ -#include "Exception.h" - -#include -#include -#include -#include -#include #include #include #include #include #include #include -#include +#include #include +#include #include #include #include +#include #include #include #include -#include +#include +#include +#include +#include +#include + +#include namespace fs = std::filesystem; diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 0c8a7177a99..97af8d1ffc3 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -1,22 +1,20 @@ #pragma once -#include -#include -#include -#include - -#include - #include #include #include #include -#include #include #include #include +#include +#include +#include +#include + #include +#include namespace Poco { class Logger; } @@ -24,6 +22,8 @@ namespace Poco { class Logger; } namespace DB { +class AtomicLogger; + [[noreturn]] void abortOnFailedAssertion(const String & description); /// This flag can be set for testing purposes - to check that no exceptions are thrown. diff --git a/src/Common/Jemalloc.cpp b/src/Common/Jemalloc.cpp index 3eb8691a1e1..6514639e700 100644 --- a/src/Common/Jemalloc.cpp +++ b/src/Common/Jemalloc.cpp @@ -2,6 +2,7 @@ #if USE_JEMALLOC +#include #include #include #include diff --git a/src/Common/Logger.h b/src/Common/Logger.h index 0425da8c847..b4688eb0a9c 100644 --- a/src/Common/Logger.h +++ b/src/Common/Logger.h @@ -1,15 +1,20 @@ #pragma once -#include - #include -#include +#include + #include #include -using LoggerPtr = Poco::LoggerPtr; +namespace Poco +{ +class Channel; +class Logger; +using LoggerPtr = std::shared_ptr; +} +using LoggerPtr = std::shared_ptr; using LoggerRawPtr = Poco::Logger *; /** RAII wrappers around Poco/Logger.h. diff --git a/src/Common/ProfileEventsScope.cpp b/src/Common/ProfileEventsScope.cpp index 92f75f4f5b0..1ba5f7f165c 100644 --- a/src/Common/ProfileEventsScope.cpp +++ b/src/Common/ProfileEventsScope.cpp @@ -1,3 +1,4 @@ +#include #include namespace DB diff --git a/src/Common/ProfileEventsScope.h b/src/Common/ProfileEventsScope.h index 0444531d02b..8eabfcb55c0 100644 --- a/src/Common/ProfileEventsScope.h +++ b/src/Common/ProfileEventsScope.h @@ -1,7 +1,8 @@ #pragma once #include -#include + +#include namespace DB { diff --git a/src/Common/logger_useful.h b/src/Common/logger_useful.h index 1ce4f545e6f..9d6ebaddcc6 100644 --- a/src/Common/logger_useful.h +++ b/src/Common/logger_useful.h @@ -5,13 +5,11 @@ #include #include #include -#include -#include -#include -#include #include - -namespace Poco { class Logger; } +#include +#include +#include +#include #define LogToStr(x, y) std::make_unique(x, y) @@ -22,7 +20,7 @@ using LogSeriesLimiterPtr = std::shared_ptr; namespace impl { [[maybe_unused]] inline LoggerPtr getLoggerHelper(const LoggerPtr & logger) { return logger; } - [[maybe_unused]] inline LoggerPtr getLoggerHelper(const AtomicLogger & logger) { return logger.load(); } + [[maybe_unused]] inline LoggerPtr getLoggerHelper(const DB::AtomicLogger & logger) { return logger.load(); } [[maybe_unused]] inline const ::Poco::Logger * getLoggerHelper(const ::Poco::Logger * logger) { return logger; } [[maybe_unused]] inline std::unique_ptr getLoggerHelper(std::unique_ptr && logger) { return logger; } [[maybe_unused]] inline std::unique_ptr getLoggerHelper(std::unique_ptr && logger) { return logger; } @@ -66,8 +64,7 @@ namespace impl #define LOG_IMPL(logger, priority, PRIORITY, ...) do \ { \ auto _logger = ::impl::getLoggerHelper(logger); \ - const bool _is_clients_log = (DB::CurrentThread::getGroup() != nullptr) && \ - (DB::CurrentThread::get().getClientLogsLevel() >= (priority)); \ + const bool _is_clients_log = DB::currentThreadHasGroup() && DB::currentThreadLogsLevel() >= (priority); \ if (!_is_clients_log && !_logger->is((PRIORITY))) \ break; \ \ diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h index eca93353283..a1450be2466 100644 --- a/src/Core/BackgroundSchedulePool.h +++ b/src/Core/BackgroundSchedulePool.h @@ -1,21 +1,20 @@ #pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include #include -#include +#include namespace DB diff --git a/src/IO/TimeoutSetter.cpp b/src/IO/TimeoutSetter.cpp index b355a119462..2e732782700 100644 --- a/src/IO/TimeoutSetter.cpp +++ b/src/IO/TimeoutSetter.cpp @@ -1,5 +1,5 @@ #include - +#include #include diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index bd56a540128..f9e24e2de70 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -1,19 +1,20 @@ #include "ExternalLoader.h" #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace DB diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 311ece67bce..98d7a362bd7 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -1,7 +1,8 @@ -#include "CertificateReloader.h" +#include #if USE_SSL +#include #include #include #include diff --git a/src/Storages/NATS/NATSHandler.cpp b/src/Storages/NATS/NATSHandler.cpp index 03f1fc1a495..f0554a7f1f0 100644 --- a/src/Storages/NATS/NATSHandler.cpp +++ b/src/Storages/NATS/NATSHandler.cpp @@ -1,3 +1,4 @@ +#include #include #include #include From 66cd879d2dd48381249c0f3263c646e5d220d3c0 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 2 Apr 2024 13:58:17 +0200 Subject: [PATCH 0825/1165] Fix msan --- src/Coordination/Changelog.cpp | 2 +- src/Coordination/InMemoryLogStore.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index d531b1266ef..58d396aad88 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -2219,7 +2219,7 @@ uint64_t Changelog::getStartIndex() const LogEntryPtr Changelog::getLastEntry() const { /// This entry treaded in special way by NuRaft - static LogEntryPtr fake_entry = nuraft::cs_new(0, nuraft::buffer::alloc(sizeof(uint64_t))); + static LogEntryPtr fake_entry = nuraft::cs_new(0, nuraft::buffer::alloc(0)); auto entry = entry_storage.getEntry(max_log_id); if (entry == nullptr) diff --git a/src/Coordination/InMemoryLogStore.cpp b/src/Coordination/InMemoryLogStore.cpp index ee93c02b4b0..32aaf8e0d4a 100644 --- a/src/Coordination/InMemoryLogStore.cpp +++ b/src/Coordination/InMemoryLogStore.cpp @@ -16,7 +16,7 @@ ptr makeClone(const ptr & entry) InMemoryLogStore::InMemoryLogStore() : start_idx(1) { - nuraft::ptr buf = nuraft::buffer::alloc(sizeof(uint64_t)); + nuraft::ptr buf = nuraft::buffer::alloc(0); logs[0] = nuraft::cs_new(0, buf); } From 27d13a400ffde0a4d983a6a4589b3f1fa7dd3649 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 2 Apr 2024 20:39:17 +0800 Subject: [PATCH 0826/1165] add settings to changes history --- src/Core/SettingsChangesHistory.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 170836cb980..f578e0c8d0a 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -125,6 +125,7 @@ static std::map sett {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."}, {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."}, {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."}, + {"input_format_hive_text_allow_variable_number_of_columns", true, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, From c35a4364359e0d0d2b061c684b3005c5e179b5e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 2 Apr 2024 14:45:48 +0200 Subject: [PATCH 0827/1165] Remove nested dependency on DateLutImpl --- programs/client/Client.cpp | 9 +- .../AggregateFunctionFactory.cpp | 2 +- src/Analyzer/SetUtils.h | 9 +- src/Common/DateLUT.cpp | 38 +++++++- src/Common/DateLUT.h | 45 ++------- src/Common/DateLUTImpl.cpp | 34 ++++++- src/Common/DateLUTImpl.h | 30 ++---- src/Common/LocalDate.h | 3 +- src/Common/LoggingFormatStringHelpers.cpp | 1 + src/Common/QueryProfiler.cpp | 11 ++- src/Common/ThreadFuzzer.cpp | 1 + src/Common/ZooKeeper/ZooKeeperImpl.cpp | 11 ++- src/Common/mysqlxx/mysqlxx/Pool.h | 2 + src/Common/tests/gtest_resolve_pool.cpp | 3 +- src/Daemon/BaseDaemon.cpp | 6 +- src/DataTypes/DataTypeDate32.cpp | 7 ++ src/DataTypes/DataTypeDate32.h | 6 +- .../DatabaseMaterializedPostgreSQL.cpp | 1 + src/Formats/JSONUtils.h | 1 + src/Functions/DateTimeTransforms.h | 2 +- src/Functions/FunctionsConversion.cpp | 93 ++++++++++--------- src/Functions/FunctionsTimeWindow.h | 3 +- .../extractTimeZoneFromFunctionArguments.cpp | 7 +- src/Functions/makeDate.cpp | 1 + src/Functions/today.cpp | 8 +- src/Functions/yesterday.cpp | 8 +- src/IO/S3/BlobStorageLogWriter.cpp | 1 + src/IO/S3/Client.cpp | 1 + src/Interpreters/AsynchronousMetricLog.cpp | 5 +- src/Interpreters/Cache/EvictionCandidates.cpp | 1 + src/Interpreters/Cache/FileSegment.cpp | 3 +- .../Cache/LRUFileCachePriority.cpp | 11 ++- src/Interpreters/Cache/QueryLimit.cpp | 5 +- .../Cache/WriteBufferToFileSegment.cpp | 3 +- src/Interpreters/CrashLog.cpp | 11 ++- .../FilesystemReadPrefetchesLog.cpp | 3 +- src/Interpreters/HashJoin.cpp | 10 +- src/Interpreters/MetricLog.cpp | 11 ++- src/Interpreters/ProcessorsProfileLog.cpp | 9 +- src/Interpreters/tests/gtest_page_cache.cpp | 2 + src/Loggers/Loggers.cpp | 9 +- .../Formats/Impl/CHColumnToArrowColumn.cpp | 2 +- src/Server/InterserverIOHTTPHandler.cpp | 3 +- src/Server/MySQLHandler.cpp | 2 + src/Server/PostgreSQLHandler.cpp | 10 +- src/Storages/FileLog/DirectoryWatcherBase.h | 1 + src/Storages/Kafka/StorageKafka.h | 3 +- src/Storages/MaterializedView/RefreshSet.h | 2 +- .../MergeTree/BackgroundJobsAssignee.h | 4 +- src/Storages/S3Queue/S3QueueFilesMetadata.cpp | 10 +- 50 files changed, 262 insertions(+), 202 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index d4bf2f686c8..192f9e61891 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -17,12 +17,13 @@ #include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index 18edb7c8ce0..6555ae63128 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -1,11 +1,11 @@ #include #include - #include #include #include #include #include +#include static constexpr size_t MAX_AGGREGATE_FUNCTION_NAME_LENGTH = 1000; diff --git a/src/Analyzer/SetUtils.h b/src/Analyzer/SetUtils.h index c35b45dce59..aef906a6576 100644 --- a/src/Analyzer/SetUtils.h +++ b/src/Analyzer/SetUtils.h @@ -1,14 +1,15 @@ #pragma once -#include +#include -#include - -#include +#include namespace DB { +class IDataType; +using DataTypePtr = std::shared_ptr; + class Set; using SetPtr = std::shared_ptr; diff --git a/src/Common/DateLUT.cpp b/src/Common/DateLUT.cpp index 2b261a28469..3a20fb1a125 100644 --- a/src/Common/DateLUT.cpp +++ b/src/Common/DateLUT.cpp @@ -1,13 +1,15 @@ #include "DateLUT.h" +#include +#include +#include + #include #include #include -#include #include #include -#include namespace @@ -140,6 +142,38 @@ std::string determineDefaultTimeZone() } +const DateLUTImpl & DateLUT::instance() +{ + const auto & date_lut = getInstance(); + + if (DB::CurrentThread::isInitialized()) + { + std::string timezone_from_context; + const DB::ContextPtr query_context = DB::CurrentThread::get().getQueryContext(); + + if (query_context) + { + timezone_from_context = extractTimezoneFromContext(query_context); + + if (!timezone_from_context.empty()) + return date_lut.getImplementation(timezone_from_context); + } + + /// On the server side, timezone is passed in query_context, + /// but on CH-client side we have no query context, + /// and each time we modify client's global context + const DB::ContextPtr global_context = DB::CurrentThread::get().getGlobalContext(); + if (global_context) + { + timezone_from_context = extractTimezoneFromContext(global_context); + + if (!timezone_from_context.empty()) + return date_lut.getImplementation(timezone_from_context); + } + } + return serverTimezoneInstance(); +} + DateLUT::DateLUT() { /// Initialize the pointer to the default DateLUTImpl. diff --git a/src/Common/DateLUT.h b/src/Common/DateLUT.h index 2045d4895e7..d0b85ea9895 100644 --- a/src/Common/DateLUT.h +++ b/src/Common/DateLUT.h @@ -1,17 +1,23 @@ #pragma once -#include "DateLUTImpl.h" - #include +#include #include -#include "Common/CurrentThread.h" #include #include #include #include +namespace DB +{ +class Context; +using ContextPtr = std::shared_ptr; +} + +class DateLUTImpl; + /// This class provides lazy initialization and lookup of singleton DateLUTImpl objects for a given timezone. class DateLUT : private boost::noncopyable @@ -20,38 +26,7 @@ public: /// Return DateLUTImpl instance for session timezone. /// session_timezone is a session-level setting. /// If setting is not set, returns the server timezone. - static ALWAYS_INLINE const DateLUTImpl & instance() - { - const auto & date_lut = getInstance(); - - if (DB::CurrentThread::isInitialized()) - { - std::string timezone_from_context; - const DB::ContextPtr query_context = DB::CurrentThread::get().getQueryContext(); - - if (query_context) - { - timezone_from_context = extractTimezoneFromContext(query_context); - - if (!timezone_from_context.empty()) - return date_lut.getImplementation(timezone_from_context); - } - - /// On the server side, timezone is passed in query_context, - /// but on CH-client side we have no query context, - /// and each time we modify client's global context - const DB::ContextPtr global_context = DB::CurrentThread::get().getGlobalContext(); - if (global_context) - { - timezone_from_context = extractTimezoneFromContext(global_context); - - if (!timezone_from_context.empty()) - return date_lut.getImplementation(timezone_from_context); - } - - } - return serverTimezoneInstance(); - } + static const DateLUTImpl & instance(); static ALWAYS_INLINE const DateLUTImpl & instance(const std::string & time_zone) { diff --git a/src/Common/DateLUTImpl.cpp b/src/Common/DateLUTImpl.cpp index bb677b3a62d..341e571e4eb 100644 --- a/src/Common/DateLUTImpl.cpp +++ b/src/Common/DateLUTImpl.cpp @@ -1,8 +1,5 @@ -#include "DateLUTImpl.h" - -#include -#include -#include +#include +#include #include #include @@ -11,6 +8,10 @@ #include #include +#include +#include +#include + namespace DB { @@ -214,6 +215,29 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) } } +unsigned int DateLUTImpl::toMillisecond(const DB::DateTime64 & datetime, Int64 scale_multiplier) const +{ + constexpr Int64 millisecond_multiplier = 1'000; + constexpr Int64 microsecond_multiplier = 1'000 * millisecond_multiplier; + constexpr Int64 divider = microsecond_multiplier / millisecond_multiplier; + + auto components = DB::DecimalUtils::splitWithScaleMultiplier(datetime, scale_multiplier); + + if (datetime.value < 0 && components.fractional) + { + components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional; + --components.whole; + } + Int64 fractional = components.fractional; + if (scale_multiplier > microsecond_multiplier) + fractional = fractional / (scale_multiplier / microsecond_multiplier); + else if (scale_multiplier < microsecond_multiplier) + fractional = fractional * (microsecond_multiplier / scale_multiplier); + + UInt16 millisecond = static_cast(fractional / divider); + return millisecond; +} + /// Prefer to load timezones from blobs linked to the binary. /// The blobs are provided by "tzdata" library. diff --git a/src/Common/DateLUTImpl.h b/src/Common/DateLUTImpl.h index 082127e717c..01cbae3d447 100644 --- a/src/Common/DateLUTImpl.h +++ b/src/Common/DateLUTImpl.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -50,6 +49,11 @@ enum class WeekDayMode WeekStartsSunday1 = 3 }; +namespace DB +{ +class DateTime64; +} + /** Lookup table to conversion of time to date, and to month / year / day of week / day of month and so on. * First time was implemented for OLAPServer, that needed to do billions of such transformations. */ @@ -593,29 +597,7 @@ public: return time % 60; } - template - unsigned toMillisecond(const DateOrTime & datetime, Int64 scale_multiplier) const - { - constexpr Int64 millisecond_multiplier = 1'000; - constexpr Int64 microsecond_multiplier = 1'000 * millisecond_multiplier; - constexpr Int64 divider = microsecond_multiplier / millisecond_multiplier; - - auto components = DB::DecimalUtils::splitWithScaleMultiplier(datetime, scale_multiplier); - - if (datetime.value < 0 && components.fractional) - { - components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional; - --components.whole; - } - Int64 fractional = components.fractional; - if (scale_multiplier > microsecond_multiplier) - fractional = fractional / (scale_multiplier / microsecond_multiplier); - else if (scale_multiplier < microsecond_multiplier) - fractional = fractional * (microsecond_multiplier / scale_multiplier); - - UInt16 millisecond = static_cast(fractional / divider); - return millisecond; - } + unsigned toMillisecond(const DB::DateTime64 & datetime, Int64 scale_multiplier) const; unsigned toMinute(Time t) const { diff --git a/src/Common/LocalDate.h b/src/Common/LocalDate.h index 2331a40fd12..f1abc98c8e2 100644 --- a/src/Common/LocalDate.h +++ b/src/Common/LocalDate.h @@ -1,9 +1,10 @@ #pragma once #include -#include #include +#include #include +#include /** Stores a calendar date in broken-down form (year, month, day-in-month). diff --git a/src/Common/LoggingFormatStringHelpers.cpp b/src/Common/LoggingFormatStringHelpers.cpp index 7cbef779f28..3e90526f76d 100644 --- a/src/Common/LoggingFormatStringHelpers.cpp +++ b/src/Common/LoggingFormatStringHelpers.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 34ffbf6c498..61d4d7d609c 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -1,15 +1,16 @@ #include "QueryProfiler.h" #include -#include +#include +#include +#include #include #include +#include #include -#include +#include #include -#include -#include -#include +#include #include diff --git a/src/Common/ThreadFuzzer.cpp b/src/Common/ThreadFuzzer.cpp index 9f9ec4fa356..d1e252a8184 100644 --- a/src/Common/ThreadFuzzer.cpp +++ b/src/Common/ThreadFuzzer.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 03d1b5a93d4..2185d32e47a 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1,7 +1,8 @@ #include -#include -#include +#include +#include +#include #include #include #include @@ -10,17 +11,17 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include -#include -#include -#include +#include #include "Coordination/KeeperConstants.h" #include "config.h" diff --git a/src/Common/mysqlxx/mysqlxx/Pool.h b/src/Common/mysqlxx/mysqlxx/Pool.h index c85295c4dd0..6e509d8bdd6 100644 --- a/src/Common/mysqlxx/mysqlxx/Pool.h +++ b/src/Common/mysqlxx/mysqlxx/Pool.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include diff --git a/src/Common/tests/gtest_resolve_pool.cpp b/src/Common/tests/gtest_resolve_pool.cpp index 25e867fdebc..eef4635e7b1 100644 --- a/src/Common/tests/gtest_resolve_pool.cpp +++ b/src/Common/tests/gtest_resolve_pool.cpp @@ -1,6 +1,7 @@ #include -#include #include +#include +#include #include #include diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 7fc210a691a..cc22db3969c 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -1,9 +1,11 @@ #pragma clang diagnostic ignored "-Wreserved-identifier" +#include +#include +#include +#include #include #include -#include -#include #include #include diff --git a/src/DataTypes/DataTypeDate32.cpp b/src/DataTypes/DataTypeDate32.cpp index 83b1260eb6d..762552bcb4c 100644 --- a/src/DataTypes/DataTypeDate32.cpp +++ b/src/DataTypes/DataTypeDate32.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include namespace DB { @@ -14,6 +16,11 @@ SerializationPtr DataTypeDate32::doGetDefaultSerialization() const return std::make_shared(); } +Field DataTypeDate32::getDefault() const +{ + return -static_cast(DateLUT::instance().getDayNumOffsetEpoch()); +} + void registerDataTypeDate32(DataTypeFactory & factory) { factory.registerSimpleDataType( diff --git a/src/DataTypes/DataTypeDate32.h b/src/DataTypes/DataTypeDate32.h index 02e818f10df..65633e7a228 100644 --- a/src/DataTypes/DataTypeDate32.h +++ b/src/DataTypes/DataTypeDate32.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include namespace DB @@ -15,10 +14,7 @@ public: TypeIndex getColumnType() const override { return TypeIndex::Int32; } const char * getFamilyName() const override { return family_name; } - Field getDefault() const override - { - return -static_cast(DateLUT::instance().getDayNumOffsetEpoch()); - } + Field getDefault() const override; bool canBeUsedAsVersion() const override { return true; } bool canBeInsideNullable() const override { return true; } diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index b44bc136b1f..1c0d5fe3de1 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/src/Formats/JSONUtils.h b/src/Formats/JSONUtils.h index 106884484c7..7ee111c1285 100644 --- a/src/Formats/JSONUtils.h +++ b/src/Formats/JSONUtils.h @@ -13,6 +13,7 @@ namespace DB { +class Block; struct JSONInferenceInfo; namespace JSONUtils diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 558c309007c..4e8c1d651e1 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -1526,7 +1526,7 @@ struct ToMillisecondImpl static UInt16 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl & time_zone) { - return time_zone.toMillisecond(datetime64, scale_multiplier); + return time_zone.toMillisecond(datetime64, scale_multiplier); } static UInt16 execute(UInt32, const DateLUTImpl &) diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 7049ca44110..0f624a2fa2e 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -1,66 +1,67 @@ #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include #include -#include -#include -#include +#include +#include #include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include #include -#include -#include +#include +#include +#include +#include +#include #include +#include +#include +#include #include -#include #include -#include +#include +#include namespace DB diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index d52b76bec91..4532286830d 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -1,8 +1,9 @@ #pragma once -#include #include #include +#include +#include namespace DB diff --git a/src/Functions/extractTimeZoneFromFunctionArguments.cpp b/src/Functions/extractTimeZoneFromFunctionArguments.cpp index 7168c68c9c9..cb8a834ed3b 100644 --- a/src/Functions/extractTimeZoneFromFunctionArguments.cpp +++ b/src/Functions/extractTimeZoneFromFunctionArguments.cpp @@ -1,10 +1,11 @@ -#include -#include +#include #include #include #include -#include +#include +#include #include +#include namespace DB diff --git a/src/Functions/makeDate.cpp b/src/Functions/makeDate.cpp index c7f3c195578..8794283a856 100644 --- a/src/Functions/makeDate.cpp +++ b/src/Functions/makeDate.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include diff --git a/src/Functions/today.cpp b/src/Functions/today.cpp index 16a5b98d7ec..356660fa7b5 100644 --- a/src/Functions/today.cpp +++ b/src/Functions/today.cpp @@ -1,11 +1,9 @@ -#include - #include - #include - -#include #include +#include +#include +#include namespace DB diff --git a/src/Functions/yesterday.cpp b/src/Functions/yesterday.cpp index 43832c1faaa..4d79f1fef79 100644 --- a/src/Functions/yesterday.cpp +++ b/src/Functions/yesterday.cpp @@ -1,11 +1,9 @@ -#include - #include - #include - -#include #include +#include +#include +#include namespace DB diff --git a/src/IO/S3/BlobStorageLogWriter.cpp b/src/IO/S3/BlobStorageLogWriter.cpp index fe33f1c8799..aaf4aea5a8e 100644 --- a/src/IO/S3/BlobStorageLogWriter.cpp +++ b/src/IO/S3/BlobStorageLogWriter.cpp @@ -3,6 +3,7 @@ #if USE_AWS_S3 #include +#include #include #include #include diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 4f93aba2f84..b2ad4668095 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -1,4 +1,5 @@ #include +#include #include #if USE_AWS_S3 diff --git a/src/Interpreters/AsynchronousMetricLog.cpp b/src/Interpreters/AsynchronousMetricLog.cpp index dc67bd91550..4287798c4ca 100644 --- a/src/Interpreters/AsynchronousMetricLog.cpp +++ b/src/Interpreters/AsynchronousMetricLog.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,8 +5,10 @@ #include #include #include -#include #include +#include +#include +#include namespace DB diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index cb15af72704..f21c5f3a508 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace ProfileEvents diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 9ec2b090dc7..e474e24c6f1 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -6,10 +6,11 @@ #include #include #include +#include +#include #include #include #include -#include #include #include diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index e65c102f1e3..ddc30755409 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -1,10 +1,11 @@ -#include -#include #include -#include -#include -#include +#include +#include #include +#include +#include +#include +#include namespace CurrentMetrics { diff --git a/src/Interpreters/Cache/QueryLimit.cpp b/src/Interpreters/Cache/QueryLimit.cpp index 9421005dc92..6a5b5bf67ca 100644 --- a/src/Interpreters/Cache/QueryLimit.cpp +++ b/src/Interpreters/Cache/QueryLimit.cpp @@ -1,6 +1,7 @@ -#include -#include #include +#include +#include +#include namespace DB { diff --git a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp index 51914c0a14e..acdfa0d5437 100644 --- a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp +++ b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp @@ -7,8 +7,9 @@ #include -#include +#include #include +#include namespace DB { diff --git a/src/Interpreters/CrashLog.cpp b/src/Interpreters/CrashLog.cpp index 410ea922429..4a8ef84fd5c 100644 --- a/src/Interpreters/CrashLog.cpp +++ b/src/Interpreters/CrashLog.cpp @@ -1,14 +1,15 @@ -#include -#include #include +#include +#include #include #include #include -#include -#include +#include +#include #include -#include +#include #include +#include #include diff --git a/src/Interpreters/FilesystemReadPrefetchesLog.cpp b/src/Interpreters/FilesystemReadPrefetchesLog.cpp index 7fb2e3d1f4c..8cea05a1857 100644 --- a/src/Interpreters/FilesystemReadPrefetchesLog.cpp +++ b/src/Interpreters/FilesystemReadPrefetchesLog.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,6 +5,8 @@ #include #include #include +#include +#include namespace DB diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index ddd65f95627..12a906526f6 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -3,15 +3,15 @@ #include #include -#include -#include - #include -#include -#include #include #include +#include #include +#include +#include +#include +#include #include diff --git a/src/Interpreters/MetricLog.cpp b/src/Interpreters/MetricLog.cpp index 5f6db0da520..6ed29cfadcb 100644 --- a/src/Interpreters/MetricLog.cpp +++ b/src/Interpreters/MetricLog.cpp @@ -1,12 +1,13 @@ -#include -#include -#include -#include -#include #include #include #include +#include #include +#include +#include +#include +#include +#include namespace DB diff --git a/src/Interpreters/ProcessorsProfileLog.cpp b/src/Interpreters/ProcessorsProfileLog.cpp index 015b4abc712..7dec2a3163a 100644 --- a/src/Interpreters/ProcessorsProfileLog.cpp +++ b/src/Interpreters/ProcessorsProfileLog.cpp @@ -1,15 +1,16 @@ #include -#include -#include +#include #include #include #include #include +#include #include #include -#include -#include +#include +#include +#include #include #include diff --git a/src/Interpreters/tests/gtest_page_cache.cpp b/src/Interpreters/tests/gtest_page_cache.cpp index 1e2688c0ca2..30fa3b921c9 100644 --- a/src/Interpreters/tests/gtest_page_cache.cpp +++ b/src/Interpreters/tests/gtest_page_cache.cpp @@ -1,4 +1,6 @@ +#include #include + #include #include diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index cc6e4691737..c5862b82f34 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -1,14 +1,17 @@ #include "Loggers.h" -#include -#include -#include #include "OwnFormattingChannel.h" #include "OwnPatternFormatter.h" #include "OwnSplitChannel.h" + +#include +#include + #include #include #include +#include +#include #ifndef WITHOUT_TEXT_LOG #include diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 9d6c8420069..2b40e796c5c 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -2,7 +2,6 @@ #if USE_ARROW || USE_PARQUET -// #include #include #include #include @@ -10,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index 28045380cd7..0d79aaa227b 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -8,8 +8,9 @@ #include #include #include -#include +#include #include +#include #include #include diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 32490627214..6456f6d24ff 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -21,8 +21,10 @@ #include #include #include +#include #include #include +#include #include #include #include diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 83e06628185..473d681ddb2 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -1,17 +1,19 @@ +#include "PostgreSQLHandler.h" #include -#include #include +#include #include #include #include -#include "PostgreSQLHandler.h" #include #include -#include -#include #include #include +#include +#include #include +#include +#include #if USE_SSL # include diff --git a/src/Storages/FileLog/DirectoryWatcherBase.h b/src/Storages/FileLog/DirectoryWatcherBase.h index 0dfb58fbc5c..3bf93415b8f 100644 --- a/src/Storages/FileLog/DirectoryWatcherBase.h +++ b/src/Storages/FileLog/DirectoryWatcherBase.h @@ -2,6 +2,7 @@ #include #include +#include #include #include diff --git a/src/Storages/Kafka/StorageKafka.h b/src/Storages/Kafka/StorageKafka.h index 829e23faf77..6479902f0f2 100644 --- a/src/Storages/Kafka/StorageKafka.h +++ b/src/Storages/Kafka/StorageKafka.h @@ -19,8 +19,9 @@ namespace DB { -class StorageSystemKafkaConsumers; class ReadFromStorageKafka; +class StorageSystemKafkaConsumers; +class ThreadStatus; struct StorageKafkaInterceptors; diff --git a/src/Storages/MaterializedView/RefreshSet.h b/src/Storages/MaterializedView/RefreshSet.h index 792e4b60897..eff445023a6 100644 --- a/src/Storages/MaterializedView/RefreshSet.h +++ b/src/Storages/MaterializedView/RefreshSet.h @@ -1,9 +1,9 @@ #pragma once +#include #include #include #include - #include namespace DB diff --git a/src/Storages/MergeTree/BackgroundJobsAssignee.h b/src/Storages/MergeTree/BackgroundJobsAssignee.h index 65fefce0917..9369ebe9135 100644 --- a/src/Storages/MergeTree/BackgroundJobsAssignee.h +++ b/src/Storages/MergeTree/BackgroundJobsAssignee.h @@ -1,7 +1,9 @@ #pragma once -#include #include +#include +#include + #include diff --git a/src/Storages/S3Queue/S3QueueFilesMetadata.cpp b/src/Storages/S3Queue/S3QueueFilesMetadata.cpp index 7b4438e1387..ed2f8d2ec1b 100644 --- a/src/Storages/S3Queue/S3QueueFilesMetadata.cpp +++ b/src/Storages/S3Queue/S3QueueFilesMetadata.cpp @@ -1,9 +1,5 @@ #include "config.h" -#include -#include -#include -#include #include #include #include @@ -12,6 +8,12 @@ #include #include #include +#include +#include +#include +#include +#include + #include #include #include From 9c3bf05675700a7ad2c9c66bb4ea67aaf2d8964f Mon Sep 17 00:00:00 2001 From: yariks5s Date: Tue, 2 Apr 2024 14:58:45 +0000 Subject: [PATCH 0828/1165] init --- .../sql-reference/aggregate-functions/parametric-functions.md | 2 +- .../sql-reference/aggregate-functions/parametric-functions.md | 2 +- .../sql-reference/aggregate-functions/parametric-functions.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 3654cd157e9..38dca6b7071 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -483,7 +483,7 @@ Where: - `r1`- the number of unique visitors who visited the site during 2020-01-01 (the `cond1` condition). - `r2`- the number of unique visitors who visited the site during a specific time period between 2020-01-01 and 2020-01-02 (`cond1` and `cond2` conditions). -- `r3`- the number of unique visitors who visited the site during a specific time period between 2020-01-01 and 2020-01-03 (`cond1` and `cond3` conditions). +- `r3`- the number of unique visitors who visited the site during a specific time period on 2020-01-01 and 2020-01-03 (`cond1` and `cond3` conditions). ## uniqUpTo(N)(x) diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index 59a9c7f8cf1..6463f6bd95d 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -476,7 +476,7 @@ FROM - `r1` - количество уникальных посетителей за 2020-01-01 (`cond1`). - `r2` - количество уникальных посетителей в период между 2020-01-01 и 2020-01-02 (`cond1` и `cond2`). -- `r3` - количество уникальных посетителей в период между 2020-01-01 и 2020-01-03 (`cond1` и `cond3`). +- `r3` - количество уникальных посетителей в период за 2020-01-01 и 2020-01-03 (`cond1` и `cond3`). ## uniqUpTo(N)(x) {#uniquptonx} diff --git a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md index 1c7de515c58..cb1dcc35f5c 100644 --- a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md @@ -472,7 +472,7 @@ FROM - `r1`-2020-01-01期间访问该网站的独立访问者数量（ `cond1` 条件）。 - `r2`-在2020-01-01和2020-01-02之间的特定时间段内访问该网站的唯一访问者的数量 (`cond1` 和 `cond2` 条件）。 -- `r3`-在2020-01-01和2020-01-03之间的特定时间段内访问该网站的唯一访问者的数量 (`cond1` 和 `cond3` 条件）。 +- `r3`-在2020-01-01和2020-01-03 网站的独立访客数量 (`cond1` 和 `cond3` 条件）。 ## uniqUpTo(N)(x) {#uniquptonx} From 7285a55f6983f7d6d89e5c0e95da19ccce78e4c8 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 2 Apr 2024 17:08:32 +0200 Subject: [PATCH 0829/1165] One more --- src/Interpreters/Cache/LRUFileCachePriority.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 1d9725352be..4b65b1bd8ad 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -125,6 +125,9 @@ void LRUFileCachePriority::updateSize(int64_t size) chassert(size != 0); chassert(size > 0 || state->current_size >= size_t(-size)); + LOG_TEST(log, "Updating size with {}, current is {}", + size, state->current_size); + state->current_size += size; CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size); } From 40b9f39c00b4c4fce757540e0ea0058bbe8f8360 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 2 Apr 2024 17:47:00 +0200 Subject: [PATCH 0830/1165] Analyzer: Fix query parameters --- src/Analyzer/QueryTreeBuilder.cpp | 16 +++--- src/Analyzer/SortNode.cpp | 17 ++++--- src/Interpreters/InterpreterSelectQuery.cpp | 16 +++--- src/Interpreters/TreeOptimizer.cpp | 2 +- src/Parsers/ASTOrderByElement.cpp | 8 +-- src/Parsers/ASTOrderByElement.h | 56 ++++++++++++++++++--- src/Parsers/ExpressionElementParsers.cpp | 13 +++-- 7 files changed, 86 insertions(+), 42 deletions(-) diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index df80f46b3cd..7f7d7a82885 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -444,8 +444,8 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express nulls_sort_direction = order_by_element.nulls_direction == 1 ? SortDirection::ASCENDING : SortDirection::DESCENDING; std::shared_ptr collator; - if (order_by_element.collation) - collator = std::make_shared(order_by_element.collation->as().value.get()); + if (order_by_element.getCollation()) + collator = std::make_shared(order_by_element.getCollation()->as().value.get()); const auto & sort_expression_ast = order_by_element.children.at(0); auto sort_expression = buildExpression(sort_expression_ast, context); @@ -455,12 +455,12 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express std::move(collator), order_by_element.with_fill); - if (order_by_element.fill_from) - sort_node->getFillFrom() = buildExpression(order_by_element.fill_from, context); - if (order_by_element.fill_to) - sort_node->getFillTo() = buildExpression(order_by_element.fill_to, context); - if (order_by_element.fill_step) - sort_node->getFillStep() = buildExpression(order_by_element.fill_step, context); + if (order_by_element.getFillFrom()) + sort_node->getFillFrom() = buildExpression(order_by_element.getFillFrom(), context); + if (order_by_element.getFillTo()) + sort_node->getFillTo() = buildExpression(order_by_element.getFillTo(), context); + if (order_by_element.getFillStep()) + sort_node->getFillStep() = buildExpression(order_by_element.getFillStep(), context); list_node->getNodes().push_back(std::move(sort_node)); } diff --git a/src/Analyzer/SortNode.cpp b/src/Analyzer/SortNode.cpp index 8e9913af442..b9d93511b84 100644 --- a/src/Analyzer/SortNode.cpp +++ b/src/Analyzer/SortNode.cpp @@ -120,17 +120,18 @@ ASTPtr SortNode::toASTImpl(const ConvertToASTOptions & options) const result->nulls_direction_was_explicitly_specified = nulls_sort_direction.has_value(); - result->with_fill = with_fill; - result->fill_from = hasFillFrom() ? getFillFrom()->toAST(options) : nullptr; - result->fill_to = hasFillTo() ? getFillTo()->toAST(options) : nullptr; - result->fill_step = hasFillStep() ? getFillStep()->toAST(options) : nullptr; result->children.push_back(getExpression()->toAST(options)); if (collator) - { - result->children.push_back(std::make_shared(Field(collator->getLocale()))); - result->collation = result->children.back(); - } + result->setCollation(std::make_shared(Field(collator->getLocale()))); + + result->with_fill = with_fill; + if (hasFillFrom()) + result->setFillFrom(getFillFrom()->toAST(options)); + if (hasFillTo()) + result->setFillTo(getFillTo()->toAST(options)); + if (hasFillStep()) + result->setFillStep(getFillStep()->toAST(options)); return result; } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6bbf03bb1e0..5864b35799e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1165,13 +1165,13 @@ static FillColumnDescription getWithFillDescription(const ASTOrderByElement & or { FillColumnDescription descr; - if (order_by_elem.fill_from) - std::tie(descr.fill_from, descr.fill_from_type) = getWithFillFieldValue(order_by_elem.fill_from, context); - if (order_by_elem.fill_to) - std::tie(descr.fill_to, descr.fill_to_type) = getWithFillFieldValue(order_by_elem.fill_to, context); + if (order_by_elem.getFillFrom()) + std::tie(descr.fill_from, descr.fill_from_type) = getWithFillFieldValue(order_by_elem.getFillFrom(), context); + if (order_by_elem.getFillTo()) + std::tie(descr.fill_to, descr.fill_to_type) = getWithFillFieldValue(order_by_elem.getFillTo(), context); - if (order_by_elem.fill_step) - std::tie(descr.fill_step, descr.step_kind) = getWithFillStep(order_by_elem.fill_step, context); + if (order_by_elem.getFillStep()) + std::tie(descr.fill_step, descr.step_kind) = getWithFillStep(order_by_elem.getFillStep(), context); else descr.fill_step = order_by_elem.direction; @@ -1217,8 +1217,8 @@ SortDescription InterpreterSelectQuery::getSortDescription(const ASTSelectQuery const auto & order_by_elem = elem->as(); std::shared_ptr collator; - if (order_by_elem.collation) - collator = std::make_shared(order_by_elem.collation->as().value.get()); + if (order_by_elem.getCollation()) + collator = std::make_shared(order_by_elem.getCollation()->as().value.get()); if (order_by_elem.with_fill) { diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index b71a8e3681d..a341dae32fa 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -277,7 +277,7 @@ void optimizeDuplicatesInOrderBy(const ASTSelectQuery * select_query) const auto & order_by_elem = elem->as(); if (order_by_elem.with_fill /// Always keep elements WITH FILL as they affects other. - || elems_set.emplace(name, order_by_elem.collation ? order_by_elem.collation->getColumnName() : "").second) + || elems_set.emplace(name, order_by_elem.getCollation() ? order_by_elem.getCollation()->getColumnName() : "").second) unique_elems.emplace_back(elem); } diff --git a/src/Parsers/ASTOrderByElement.cpp b/src/Parsers/ASTOrderByElement.cpp index 318849812aa..be0416359a1 100644 --- a/src/Parsers/ASTOrderByElement.cpp +++ b/src/Parsers/ASTOrderByElement.cpp @@ -31,7 +31,7 @@ void ASTOrderByElement::formatImpl(const FormatSettings & settings, FormatState << (settings.hilite ? hilite_none : ""); } - if (collation) + if (auto collation = getCollation()) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " COLLATE " << (settings.hilite ? hilite_none : ""); collation->formatImpl(settings, state, frame); @@ -40,17 +40,17 @@ void ASTOrderByElement::formatImpl(const FormatSettings & settings, FormatState if (with_fill) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " WITH FILL" << (settings.hilite ? hilite_none : ""); - if (fill_from) + if (auto fill_from = getFillFrom()) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : ""); fill_from->formatImpl(settings, state, frame); } - if (fill_to) + if (auto fill_to = getFillTo()) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : ""); fill_to->formatImpl(settings, state, frame); } - if (fill_step) + if (auto fill_step = getFillStep()) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " STEP " << (settings.hilite ? hilite_none : ""); fill_step->formatImpl(settings, state, frame); diff --git a/src/Parsers/ASTOrderByElement.h b/src/Parsers/ASTOrderByElement.h index 4cebc30be31..6edf84d7bde 100644 --- a/src/Parsers/ASTOrderByElement.h +++ b/src/Parsers/ASTOrderByElement.h @@ -10,18 +10,34 @@ namespace DB */ class ASTOrderByElement : public IAST { +private: + enum class Child : uint8_t + { + EXPRESSION, + COLLATION, + FILL_FROM, + FILL_TO, + FILL_STEP, + }; + public: int direction = 0; /// 1 for ASC, -1 for DESC int nulls_direction = 0; /// Same as direction for NULLS LAST, opposite for NULLS FIRST. bool nulls_direction_was_explicitly_specified = false; - /** Collation for locale-specific string comparison. If empty, then sorting done by bytes. */ - ASTPtr collation; - bool with_fill = false; - ASTPtr fill_from; - ASTPtr fill_to; - ASTPtr fill_step; + + /** Collation for locale-specific string comparison. If empty, then sorting done by bytes. */ + void setCollation(ASTPtr node) { setChild(Child::COLLATION, node); } + void setFillFrom(ASTPtr node) { setChild(Child::FILL_FROM, node); } + void setFillTo(ASTPtr node) { setChild(Child::FILL_TO, node); } + void setFillStep(ASTPtr node) { setChild(Child::FILL_STEP, node); } + + /** Collation for locale-specific string comparison. If empty, then sorting done by bytes. */ + ASTPtr getCollation() const { return getChild(Child::COLLATION); } + ASTPtr getFillFrom() const { return getChild(Child::FILL_FROM); } + ASTPtr getFillTo() const { return getChild(Child::FILL_TO); } + ASTPtr getFillStep() const { return getChild(Child::FILL_STEP); } String getID(char) const override { return "OrderByElement"; } @@ -36,6 +52,34 @@ public: protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; +private: + + ASTPtr getChild(Child child) const + { + auto it = positions.find(child); + if (it != positions.end()) + return children[it->second]; + return {}; + } + + void setChild(Child child, ASTPtr node) + { + if (node == nullptr) + return; + + auto it = positions.find(child); + if (it != positions.end()) + { + children[it->second] = node; + } + else + { + positions[child] = children.size(); + children.push_back(node); + } + } + + std::unordered_map positions; }; } diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 67f4a306292..d4ad210b315 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -2120,17 +2120,16 @@ bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expect auto elem = std::make_shared(); + elem->children.push_back(expr_elem); + elem->direction = direction; elem->nulls_direction = nulls_direction; elem->nulls_direction_was_explicitly_specified = nulls_direction_was_explicitly_specified; - elem->collation = locale_node; + elem->setCollation(locale_node); elem->with_fill = has_with_fill; - elem->fill_from = fill_from; - elem->fill_to = fill_to; - elem->fill_step = fill_step; - elem->children.push_back(expr_elem); - if (locale_node) - elem->children.push_back(locale_node); + elem->setFillFrom(fill_from); + elem->setFillTo(fill_to); + elem->setFillStep(fill_step); node = elem; From ed9ee5ab4cfa56cd615024d20f1d7d1c31b88be3 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 15:47:48 +0000 Subject: [PATCH 0831/1165] First portion --- .../03033_with_fill_interpolate.reference | 0 .../03033_with_fill_interpolate.sql | 27 ++++++++++++ .../03034_normalized_ast.reference | 0 .../0_stateless/03034_normalized_ast.sql | 7 +++ ...035_alias_column_bug_distributed.reference | 0 .../03035_alias_column_bug_distributed.sql | 43 +++++++++++++++++++ .../0_stateless/03036_with_numbers.reference | 20 +++++++++ .../0_stateless/03036_with_numbers.sql | 8 ++++ 8 files changed, 105 insertions(+) create mode 100644 tests/queries/0_stateless/03033_with_fill_interpolate.reference create mode 100644 tests/queries/0_stateless/03033_with_fill_interpolate.sql create mode 100644 tests/queries/0_stateless/03034_normalized_ast.reference create mode 100644 tests/queries/0_stateless/03034_normalized_ast.sql create mode 100644 tests/queries/0_stateless/03035_alias_column_bug_distributed.reference create mode 100644 tests/queries/0_stateless/03035_alias_column_bug_distributed.sql create mode 100644 tests/queries/0_stateless/03036_with_numbers.reference create mode 100644 tests/queries/0_stateless/03036_with_numbers.sql diff --git a/tests/queries/0_stateless/03033_with_fill_interpolate.reference b/tests/queries/0_stateless/03033_with_fill_interpolate.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03033_with_fill_interpolate.sql b/tests/queries/0_stateless/03033_with_fill_interpolate.sql new file mode 100644 index 00000000000..816633af757 --- /dev/null +++ b/tests/queries/0_stateless/03033_with_fill_interpolate.sql @@ -0,0 +1,27 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/55794 +DROP TABLE IF EXISTS 03033_example_table; + +CREATE TABLE 03033_example_table +( + ColumnA Int64, + ColumnB Int64, + ColumnC Int64 +) +ENGINE = MergeTree() +ORDER BY ColumnA; + +WITH +helper AS ( + SELECT + * + FROM + 03033_example_table + ORDER BY + ColumnA WITH FILL INTERPOLATE ( + ColumnB AS ColumnC, + ColumnC AS ColumnA + ) +) +SELECT ColumnB FROM helper; + +DROP TABLE IF EXISTS 03033_example_table; diff --git a/tests/queries/0_stateless/03034_normalized_ast.reference b/tests/queries/0_stateless/03034_normalized_ast.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03034_normalized_ast.sql b/tests/queries/0_stateless/03034_normalized_ast.sql new file mode 100644 index 00000000000..ff6f8da118c --- /dev/null +++ b/tests/queries/0_stateless/03034_normalized_ast.sql @@ -0,0 +1,7 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/49472 +SELECT + concat(database, table) AS name, + count() +FROM clusterAllReplicas(default, system.tables) +GROUP BY name +FORMAT Null; diff --git a/tests/queries/0_stateless/03035_alias_column_bug_distributed.reference b/tests/queries/0_stateless/03035_alias_column_bug_distributed.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql new file mode 100644 index 00000000000..fb459b3289b --- /dev/null +++ b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql @@ -0,0 +1,43 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/44414 +DROP TABLE IF EXISTS alias_bug; +DROP TABLE IF EXISTS alias_bug_dist; +CREATE TABLE alias_bug +( + `src` String, + `theAlias` String ALIAS trimBoth(src) +) +ENGINE = MergeTree() +ORDER BY src; + +CREATE TABLE alias_bug_dist +AS alias_bug +ENGINE = Distributed('default', currentDatabase(), 'alias_bug', rand()); + +INSERT INTO alias_bug VALUES ('SOURCE1'); + +-- OK +SELECT theAlias,CAST(NULL, 'Nullable(String)') AS src FROM alias_bug LIMIT 1 FORMAT Null; + +-- Not OK +SELECT theAlias,CAST(NULL, 'Nullable(String)') AS src FROM alias_bug_dist LIMIT 1 FORMAT Null; + +DROP TABLE IF EXISTS alias_bug; +DROP TABLE IF EXISTS alias_bug_dist; +CREATE TABLE alias_bug +( + `s` String, + `src` String, + `theAlias` String ALIAS trimBoth(src) +) +ENGINE = MergeTree() +ORDER BY src; + +CREATE TABLE alias_bug_dist +AS alias_bug +ENGINE = Distributed('default', currentDatabase(), 'alias_bug', rand()); + +-- Unknown identifier +SELECT CAST(123, 'String') AS src,theAlias FROM alias_bug_dist LIMIT 1 FORMAT Null; + +DROP TABLE IF EXISTS alias_bug; +DROP TABLE IF EXISTS alias_bug_dist; diff --git a/tests/queries/0_stateless/03036_with_numbers.reference b/tests/queries/0_stateless/03036_with_numbers.reference new file mode 100644 index 00000000000..7b36cc96f5e --- /dev/null +++ b/tests/queries/0_stateless/03036_with_numbers.reference @@ -0,0 +1,20 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 diff --git a/tests/queries/0_stateless/03036_with_numbers.sql b/tests/queries/0_stateless/03036_with_numbers.sql new file mode 100644 index 00000000000..5e08bb6e065 --- /dev/null +++ b/tests/queries/0_stateless/03036_with_numbers.sql @@ -0,0 +1,8 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/13843 +WITH 10 AS n +SELECT * +FROM numbers(n); + +WITH cast(10, 'UInt64') AS n +SELECT * +FROM numbers(n); From 259da73b17189c25f70e0e15bd4bc47f1362166a Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 2 Apr 2024 17:54:49 +0200 Subject: [PATCH 0832/1165] Add a test --- .../0_stateless/03033_analyzer_query_parameters.reference | 2 ++ .../0_stateless/03033_analyzer_query_parameters.sh | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03033_analyzer_query_parameters.reference create mode 100755 tests/queries/0_stateless/03033_analyzer_query_parameters.sh diff --git a/tests/queries/0_stateless/03033_analyzer_query_parameters.reference b/tests/queries/0_stateless/03033_analyzer_query_parameters.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/tests/queries/0_stateless/03033_analyzer_query_parameters.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/03033_analyzer_query_parameters.sh b/tests/queries/0_stateless/03033_analyzer_query_parameters.sh new file mode 100755 index 00000000000..c821791e437 --- /dev/null +++ b/tests/queries/0_stateless/03033_analyzer_query_parameters.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +clickhouse-local --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 1" +clickhouse-local --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 0" From c11aa0122647b39177d381499d08f6102a5e5160 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 2 Apr 2024 17:48:48 +0200 Subject: [PATCH 0833/1165] Fix race --- src/Interpreters/Cache/EvictionCandidates.cpp | 16 +++++++++++++--- src/Interpreters/Cache/EvictionCandidates.h | 4 ++-- src/Interpreters/Cache/FileSegment.cpp | 5 +++++ src/Interpreters/Cache/FileSegment.h | 1 + src/Interpreters/Cache/LRUFileCachePriority.cpp | 6 ++++++ src/Interpreters/Cache/LRUFileCachePriority.h | 2 +- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 080ef675918..4ca6aeea22e 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -30,6 +30,10 @@ EvictionCandidates::~EvictionCandidates() iterator->invalidate(); } + /// We cannot reset evicting flag if we already removed queue entries. + if (removed_queue_entries) + return; + /// Here `candidates` contain only those file segments /// which failed to be removed during evict() /// because there was some exception before evict() @@ -62,9 +66,15 @@ void EvictionCandidates::removeQueueEntries(const CachePriorityGuard::Lock & loc for (const auto & [key, key_candidates] : candidates) { for (const auto & candidate : key_candidates.candidates) + { + const auto & file_segment = candidate->file_segment; + auto file_segment_lock = file_segment->lock(); + candidate->getQueueIterator()->remove(lock); + file_segment->setQueueIteratorUnlocked(nullptr, file_segment_lock); + } } - invalidated_queue_entries = true; + removed_queue_entries = true; } void EvictionCandidates::evict() @@ -74,7 +84,7 @@ void EvictionCandidates::evict() auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::FilesystemCacheEvictMicroseconds); - if (!invalidated_queue_entries) + if (!removed_queue_entries) queue_entries_to_invalidate.reserve(candidates_size); for (auto & [key, key_candidates] : candidates) @@ -123,7 +133,7 @@ void EvictionCandidates::evict() /// it was freed in favour of some reserver, so we can make it visibly /// free only for that particular reserver. - if (!invalidated_queue_entries) + if (!removed_queue_entries) queue_entries_to_invalidate.push_back(iterator); key_candidates.candidates.pop_back(); diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index 571010a14bc..140728ae704 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -4,7 +4,7 @@ namespace DB { -class EvictionCandidates +class EvictionCandidates : private boost::noncopyable { public: using FinalizeEvictionFunc = std::function; @@ -60,7 +60,7 @@ private: std::vector on_finalize; std::vector queue_entries_to_invalidate; - bool invalidated_queue_entries = false; + bool removed_queue_entries = false; IFileCachePriority::HoldSpacePtr hold_space; }; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 9ec2b090dc7..0d64b602928 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -165,6 +165,11 @@ FileSegment::Priority::IteratorPtr FileSegment::getQueueIterator() const void FileSegment::setQueueIterator(Priority::IteratorPtr iterator) { auto lock = lockFileSegment(); + setQueueIteratorUnlocked(iterator, lock); +} + +void FileSegment::setQueueIteratorUnlocked(Priority::IteratorPtr iterator, const FileSegmentGuard::Lock &) +{ if (queue_iterator) throw Exception(ErrorCodes::LOGICAL_ERROR, "Queue iterator cannot be set twice"); queue_iterator = iterator; diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index c34ee064345..12046e59bd6 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -176,6 +176,7 @@ public: Priority::IteratorPtr getQueueIterator() const; void setQueueIterator(Priority::IteratorPtr iterator); + void setQueueIteratorUnlocked(Priority::IteratorPtr iterator, const FileSegmentGuard::Lock &); KeyMetadataPtr tryGetKeyMetadata() const; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 78ece5a3124..4f2b17ea104 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -431,6 +431,12 @@ bool LRUFileCachePriority::modifySizeLimits( return true; } +IFileCachePriority::EntryPtr LRUFileCachePriority::LRUIterator::getEntry() const +{ + assertValid(); + return *iterator; +} + void LRUFileCachePriority::LRUIterator::remove(const CachePriorityGuard::Lock & lock) { assertValid(); diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index ed3a455126a..0d10f22701e 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -146,7 +146,7 @@ public: LRUIterator & operator =(const LRUIterator & other); bool operator ==(const LRUIterator & other) const; - EntryPtr getEntry() const override { return *iterator; } + EntryPtr getEntry() const override; size_t increasePriority(const CachePriorityGuard::Lock &) override; From 4441a1b3f3c2f6844e233e379335d4bdf4922ab5 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 15:59:17 +0000 Subject: [PATCH 0834/1165] Close https://github.com/ClickHouse/ClickHouse/issues/55803 --- .../0_stateless/03037_union_view.reference | 0 .../queries/0_stateless/03037_union_view.sql | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/queries/0_stateless/03037_union_view.reference create mode 100644 tests/queries/0_stateless/03037_union_view.sql diff --git a/tests/queries/0_stateless/03037_union_view.reference b/tests/queries/0_stateless/03037_union_view.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03037_union_view.sql b/tests/queries/0_stateless/03037_union_view.sql new file mode 100644 index 00000000000..04f4afab4c4 --- /dev/null +++ b/tests/queries/0_stateless/03037_union_view.sql @@ -0,0 +1,24 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/55803 +CREATE TABLE broken_table +( + start DateTime64(6), + end DateTime64(6), +) +ENGINE = ReplacingMergeTree(start) +ORDER BY (start); + +CREATE VIEW broken_view as +SELECT + t.start as start, + t.end as end, + cast(datediff('second', t.start, t.end) as float) as total_sec +FROM broken_table t FINAL +UNION ALL +SELECT + null as start, + null as end, + null as total_sec; + +SELECT v.start, v.total_sec +FROM broken_view v FINAL +WHERE v.start IS NOT NULL; From d6504764000b762cea48a2c633286ea77cb388c7 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 16:01:55 +0000 Subject: [PATCH 0835/1165] Better --- tests/queries/0_stateless/03037_union_view.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/03037_union_view.sql b/tests/queries/0_stateless/03037_union_view.sql index 04f4afab4c4..fb8aa7df954 100644 --- a/tests/queries/0_stateless/03037_union_view.sql +++ b/tests/queries/0_stateless/03037_union_view.sql @@ -1,4 +1,7 @@ -- https://github.com/ClickHouse/ClickHouse/issues/55803 +DROP TABLE IF EXISTS broken_table; +DROP TABLE IF EXISTS broken_view; + CREATE TABLE broken_table ( start DateTime64(6), @@ -22,3 +25,6 @@ SELECT SELECT v.start, v.total_sec FROM broken_view v FINAL WHERE v.start IS NOT NULL; + +DROP TABLE IF EXISTS broken_table; +DROP TABLE IF EXISTS broken_view; \ No newline at end of file From f5c514615301659bd9fad8b6dcc13623a034a620 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 2 Apr 2024 18:04:53 +0200 Subject: [PATCH 0836/1165] [RFC] Try to add global Real and CPU trace collector --- programs/server/Server.cpp | 6 ++- src/Common/QueryProfiler.cpp | 8 ++++ src/Common/QueryProfiler.h | 2 + src/Common/ThreadPool.cpp | 15 ++++--- src/Common/ThreadPool.h | 32 ++++++++++++--- src/Common/ThreadPool_fwd.h | 7 ++-- src/Common/ThreadStatus.h | 1 + src/Core/ServerSettings.h | 2 + src/Interpreters/ThreadStatusExt.cpp | 40 +++++++++++++++++-- .../config.d/serverwide_trace_collector.xml | 4 ++ .../__init__.py | 1 + .../configs/global_profiler.xml | 4 ++ .../test_trace_collector_serverwide/test.py | 38 ++++++++++++++++++ 13 files changed, 142 insertions(+), 18 deletions(-) create mode 100644 tests/config/config.d/serverwide_trace_collector.xml create mode 100644 tests/integration/test_trace_collector_serverwide/__init__.py create mode 100644 tests/integration/test_trace_collector_serverwide/configs/global_profiler.xml create mode 100644 tests/integration/test_trace_collector_serverwide/test.py diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 450e1696c11..a048bebc45b 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -734,13 +734,17 @@ try LOG_INFO(log, "Available CPU instruction sets: {}", cpu_info); #endif + bool will_have_trace_collector = hasPHDRCache() && config().has("trace_log"); + // Initialize global thread pool. Do it before we fetch configs from zookeeper // nodes (`from_zk`), because ZooKeeper interface uses the pool. We will // ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well. GlobalThreadPool::initialize( server_settings.max_thread_pool_size, server_settings.max_thread_pool_free_size, - server_settings.thread_pool_queue_size); + server_settings.thread_pool_queue_size, + will_have_trace_collector ? server_settings.global_profiler_real_time_period_ns : 0, + will_have_trace_collector ? server_settings.global_profiler_cpu_time_period_ns : 0); /// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed). SCOPE_EXIT({ Stopwatch watch; diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 34ffbf6c498..3b7289167e3 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -198,6 +198,7 @@ void Timer::cleanup() { if (timer_id) { + LOG_INFO(log, "CLEANUP TIMER"); int err = timer_delete(*timer_id); if (err) LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString()); @@ -260,6 +261,13 @@ QueryProfilerBase::QueryProfilerBase(UInt64 thread_id, int clock_t #endif } + +template +void QueryProfilerBase::setPeriod(UInt32 period_) +{ + timer.set(period_); +} + template QueryProfilerBase::~QueryProfilerBase() { diff --git a/src/Common/QueryProfiler.h b/src/Common/QueryProfiler.h index 254b11137cc..ea4cc73bca6 100644 --- a/src/Common/QueryProfiler.h +++ b/src/Common/QueryProfiler.h @@ -57,6 +57,8 @@ public: QueryProfilerBase(UInt64 thread_id, int clock_type, UInt32 period, int pause_signal_); ~QueryProfilerBase(); + void setPeriod(UInt32 period_); + private: void cleanup(); diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 3c2e6228421..eaee070c44f 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -490,8 +490,9 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ template class ThreadPoolImpl; -template class ThreadPoolImpl>; -template class ThreadFromGlobalPoolImpl; +template class ThreadPoolImpl>; +template class ThreadFromGlobalPoolImpl; +template class ThreadFromGlobalPoolImpl; std::unique_ptr GlobalThreadPool::the_instance; @@ -500,7 +501,9 @@ GlobalThreadPool::GlobalThreadPool( size_t max_threads_, size_t max_free_threads_, size_t queue_size_, - const bool shutdown_on_exception_) + const bool shutdown_on_exception_, + UInt64 global_profiler_real_time_period_ns_, + UInt64 global_profiler_cpu_time_period_ns_) : FreeThreadPool( CurrentMetrics::GlobalThread, CurrentMetrics::GlobalThreadActive, @@ -509,10 +512,12 @@ GlobalThreadPool::GlobalThreadPool( max_free_threads_, queue_size_, shutdown_on_exception_) + , global_profiler_real_time_period_ns(global_profiler_real_time_period_ns_) + , global_profiler_cpu_time_period_ns(global_profiler_cpu_time_period_ns_) { } -void GlobalThreadPool::initialize(size_t max_threads, size_t max_free_threads, size_t queue_size) +void GlobalThreadPool::initialize(size_t max_threads, size_t max_free_threads, size_t queue_size, UInt64 global_profiler_real_time_period_ns, UInt64 global_profiler_cpu_time_period_ns) { if (the_instance) { @@ -520,7 +525,7 @@ void GlobalThreadPool::initialize(size_t max_threads, size_t max_free_threads, s "The global thread pool is initialized twice"); } - the_instance.reset(new GlobalThreadPool(max_threads, max_free_threads, queue_size, false /*shutdown_on_exception*/)); + the_instance.reset(new GlobalThreadPool(max_threads, max_free_threads, queue_size, false /*shutdown_on_exception*/, global_profiler_real_time_period_ns, global_profiler_cpu_time_period_ns)); } GlobalThreadPool & GlobalThreadPool::instance() diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index 31e4eabf63b..528f782caf2 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -172,10 +172,21 @@ class GlobalThreadPool : public FreeThreadPool, private boost::noncopyable size_t max_threads_, size_t max_free_threads_, size_t queue_size_, - bool shutdown_on_exception_); + bool shutdown_on_exception_, + UInt64 global_profiler_real_time_period_ns_, + UInt64 global_profiler_cpu_time_period_ns_); public: - static void initialize(size_t max_threads = 10000, size_t max_free_threads = 1000, size_t queue_size = 10000); + UInt64 global_profiler_real_time_period_ns; + UInt64 global_profiler_cpu_time_period_ns; + + static void initialize( + size_t max_threads = 10000, + size_t max_free_threads = 1000, + size_t queue_size = 10000, + UInt64 global_profiler_real_time_period_ns_ = 0, + UInt64 global_profiler_cpu_time_period_ns_ = 0); + static GlobalThreadPool & instance(); static void shutdown(); }; @@ -187,7 +198,7 @@ public: * NOTE: User code should use 'ThreadFromGlobalPool' declared below instead of directly using this class. * */ -template +template class ThreadFromGlobalPoolImpl : boost::noncopyable { public: @@ -197,11 +208,15 @@ public: explicit ThreadFromGlobalPoolImpl(Function && func, Args &&... args) : state(std::make_shared()) { + UInt64 global_profiler_real_time_period = GlobalThreadPool::instance().global_profiler_real_time_period_ns; + UInt64 global_profiler_cpu_time_period = GlobalThreadPool::instance().global_profiler_cpu_time_period_ns; /// NOTE: /// - If this will throw an exception, the destructor won't be called /// - this pointer cannot be passed in the lambda, since after detach() it will not be valid GlobalThreadPool::instance().scheduleOrThrow([ my_state = state, + global_profiler_real_time_period, + global_profiler_cpu_time_period, my_func = std::forward(func), my_args = std::make_tuple(std::forward(args)...)]() mutable /// mutable is needed to destroy capture { @@ -220,6 +235,12 @@ public: /// Thread status holds raw pointer on query context, thus it always must be destroyed /// before sending signal that permits to join this thread. DB::ThreadStatus thread_status; + if constexpr (global_trace_collector_allowed) + { + if (unlikely(global_profiler_real_time_period != 0 || global_profiler_cpu_time_period != 0)) + thread_status.initGlobalProfiler(global_profiler_real_time_period, global_profiler_cpu_time_period); + } + std::apply(function, arguments); }, {}, // default priority @@ -305,11 +326,12 @@ protected: /// you need to use class, or you need to use ThreadFromGlobalPool below. /// /// See the comments of ThreadPool below to know how it works. -using ThreadFromGlobalPoolNoTracingContextPropagation = ThreadFromGlobalPoolImpl; +using ThreadFromGlobalPoolNoTracingContextPropagation = ThreadFromGlobalPoolImpl; /// An alias of thread that execute jobs/tasks on global thread pool by implicit passing tracing context on current thread to underlying worker as parent tracing context. /// If jobs/tasks are directly scheduled by using APIs of this class, you need to use this class or you need to use class above. -using ThreadFromGlobalPool = ThreadFromGlobalPoolImpl; +using ThreadFromGlobalPool = ThreadFromGlobalPoolImpl; +using ThreadFromGlobalPoolWithoutTraceCollector = ThreadFromGlobalPoolImpl; /// Recommended thread pool for the case when multiple thread pools are created and destroyed. /// diff --git a/src/Common/ThreadPool_fwd.h b/src/Common/ThreadPool_fwd.h index 2782acc9c51..fea4e59f087 100644 --- a/src/Common/ThreadPool_fwd.h +++ b/src/Common/ThreadPool_fwd.h @@ -3,11 +3,12 @@ template class ThreadPoolImpl; -template +template class ThreadFromGlobalPoolImpl; -using ThreadFromGlobalPoolNoTracingContextPropagation = ThreadFromGlobalPoolImpl; +using ThreadFromGlobalPoolNoTracingContextPropagation = ThreadFromGlobalPoolImpl; -using ThreadFromGlobalPool = ThreadFromGlobalPoolImpl; +using ThreadFromGlobalPool = ThreadFromGlobalPoolImpl; +using ThreadFromGlobalPoolWithoutTraceCollector = ThreadFromGlobalPoolImpl; using ThreadPool = ThreadPoolImpl; diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 48b52f8aa6e..2d33c0ac021 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -307,6 +307,7 @@ public: void flushUntrackedMemory(); + void initGlobalProfiler(UInt64 global_profiler_real_time_period, UInt64 global_profiler_cpu_time_period); private: void applyGlobalSettings(); void applyQuerySettings(); diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 6608a35a5a2..e05b3cf9e31 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -137,6 +137,8 @@ namespace DB M(UInt64, http_connections_soft_limit, 100, "Connections above this limit have significantly shorter time to live. The limit applies to the http connections which do not belong to any disk or storage.", 0) \ M(UInt64, http_connections_warn_limit, 1000, "Warning massages are written to the logs if number of in-use connections are higher than this limit. The limit applies to the http connections which do not belong to any disk or storage.", 0) \ M(UInt64, http_connections_store_limit, 5000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the http connections which do not belong to any disk or storage.", 0) \ + M(UInt64, global_profiler_real_time_period_ns, 0, "Period for real clock timer of global profiler (in nanoseconds). Set 0 value to turn off the real clock global profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ + M(UInt64, global_profiler_cpu_time_period_ns, 0, "Period for CPU clock timer of global profiler (in nanoseconds). Set 0 value to turn off the CPU clock global profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ /// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 1c24c4f85c9..4b9bd069bc6 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #if defined(OS_LINUX) # include @@ -457,6 +458,27 @@ void ThreadStatus::resetPerformanceCountersLastUsage() taskstats->reset(); } + +void ThreadStatus::initGlobalProfiler(UInt64 global_profiler_real_time_period, UInt64 global_profiler_cpu_time_period) +{ + + try + { + if (global_profiler_real_time_period > 0) + query_profiler_real = std::make_unique(thread_id, + /* period= */ static_cast(global_profiler_real_time_period)); + + if (global_profiler_cpu_time_period > 0) + query_profiler_cpu = std::make_unique(thread_id, + /* period= */ static_cast(global_profiler_cpu_time_period)); + } + catch (...) + { + tryLogCurrentException("ThreadStatus", "Cannot initialize GlobalProfiler"); + } + +} + void ThreadStatus::initQueryProfiler() { if (internal_thread) @@ -474,12 +496,22 @@ void ThreadStatus::initQueryProfiler() try { if (settings.query_profiler_real_time_period_ns > 0) - query_profiler_real = std::make_unique(thread_id, - /* period= */ static_cast(settings.query_profiler_real_time_period_ns)); + { + if (!query_profiler_real) + query_profiler_real = std::make_unique(thread_id, + /* period= */ static_cast(settings.query_profiler_real_time_period_ns)); + else + query_profiler_real->setPeriod(static_cast(settings.query_profiler_real_time_period_ns)); + } if (settings.query_profiler_cpu_time_period_ns > 0) - query_profiler_cpu = std::make_unique(thread_id, - /* period= */ static_cast(settings.query_profiler_cpu_time_period_ns)); + { + if (!query_profiler_cpu) + query_profiler_cpu = std::make_unique(thread_id, + /* period= */ static_cast(settings.query_profiler_cpu_time_period_ns)); + else + query_profiler_cpu->setPeriod(static_cast(settings.query_profiler_cpu_time_period_ns)); + } } catch (...) { diff --git a/tests/config/config.d/serverwide_trace_collector.xml b/tests/config/config.d/serverwide_trace_collector.xml new file mode 100644 index 00000000000..602e07469f3 --- /dev/null +++ b/tests/config/config.d/serverwide_trace_collector.xml @@ -0,0 +1,4 @@ + + 1000000000 + 1000000000 + diff --git a/tests/integration/test_trace_collector_serverwide/__init__.py b/tests/integration/test_trace_collector_serverwide/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_trace_collector_serverwide/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_trace_collector_serverwide/configs/global_profiler.xml b/tests/integration/test_trace_collector_serverwide/configs/global_profiler.xml new file mode 100644 index 00000000000..5112d267182 --- /dev/null +++ b/tests/integration/test_trace_collector_serverwide/configs/global_profiler.xml @@ -0,0 +1,4 @@ + + 10000000 + 10000000 + diff --git a/tests/integration/test_trace_collector_serverwide/test.py b/tests/integration/test_trace_collector_serverwide/test.py new file mode 100644 index 00000000000..5a7bba15fd7 --- /dev/null +++ b/tests/integration/test_trace_collector_serverwide/test.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import pytest +import time + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance("node1", main_configs=["configs/global_profiler.xml"]) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + +def test_global_thread_profiler(start_cluster): + node1.query("CREATE TABLE t (key UInt32, value String) Engine = MergeTree() ORDER BY key") + + node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") + node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") + node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") + node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") + node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") + node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") + node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") + node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") + + time.sleep(5) + + node1.query("SYSTEM FLUSH LOGS") + + assert int(node1.query("SELECT count() FROM system.trace_log where trace_type='Real' and query_id = ''").strip()) > 0 From 3d9a6e9b8e5f8864f9f8a0481439a316c5deaba7 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 16:06:02 +0000 Subject: [PATCH 0837/1165] Close https://github.com/ClickHouse/ClickHouse/issues/48308 --- .../03038_ambiguous_column.reference | 0 .../0_stateless/03038_ambiguous_column.sql | 41 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/queries/0_stateless/03038_ambiguous_column.reference create mode 100644 tests/queries/0_stateless/03038_ambiguous_column.sql diff --git a/tests/queries/0_stateless/03038_ambiguous_column.reference b/tests/queries/0_stateless/03038_ambiguous_column.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03038_ambiguous_column.sql b/tests/queries/0_stateless/03038_ambiguous_column.sql new file mode 100644 index 00000000000..69c8e52d734 --- /dev/null +++ b/tests/queries/0_stateless/03038_ambiguous_column.sql @@ -0,0 +1,41 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/48308 +DROP TABLE IF EXISTS 03038_table; + +CREATE TABLE 03038_table +( + `time` DateTime +) +ENGINE = MergeTree +ORDER BY time; + +SELECT * +FROM +( + SELECT + toUInt64(time) AS time, + toHour(03038_table.time) + FROM 03038_table +) +ORDER BY time ASC; + +WITH subquery AS ( + SELECT + toUInt64(time) AS time, + toHour(03038_table.time) + FROM 03038_table +) +SELECT * +FROM subquery +ORDER BY subquery.time ASC; + +SELECT * +FROM +( + SELECT + toUInt64(time) AS time, + toHour(03038_table.time) AS hour + FROM 03038_table +) +ORDER BY time ASC, hour; + +DROP TABLE IF EXISTS 03038_table; From 723a733c84a241de56fbf66ffc84a332c995c673 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 2 Apr 2024 18:07:02 +0200 Subject: [PATCH 0838/1165] Missing change --- tests/config/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config/install.sh b/tests/config/install.sh index 652d25a0a35..06f2f5fe902 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -67,6 +67,7 @@ ln -sf $SRC_PATH/config.d/validate_tcp_client_information.xml $DEST_SERVER_PATH/ ln -sf $SRC_PATH/config.d/zero_copy_destructive_operations.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/block_number.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/handlers.yaml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/serverwide_trace_collector.xml $DEST_SERVER_PATH/config.d/ # Not supported with fasttest. if [ "${DEST_SERVER_PATH}" = "/etc/clickhouse-server" ] From d11d10050b3315227dbf0019a86bd8fa25d9bf71 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 2 Apr 2024 18:08:04 +0200 Subject: [PATCH 0839/1165] Remove debug line --- src/Common/QueryProfiler.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 3b7289167e3..f985ec95e88 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -198,7 +198,6 @@ void Timer::cleanup() { if (timer_id) { - LOG_INFO(log, "CLEANUP TIMER"); int err = timer_delete(*timer_id); if (err) LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString()); From a54efe56450ed781e5fb101014cd460b9db6fefb Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 16:11:02 +0000 Subject: [PATCH 0840/1165] Close https://github.com/ClickHouse/ClickHouse/issues/45535 --- ...known_identifier_window_function.reference | 40 +++++++++++++++++++ ...039_unknown_identifier_window_function.sql | 34 ++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 tests/queries/0_stateless/03039_unknown_identifier_window_function.reference create mode 100644 tests/queries/0_stateless/03039_unknown_identifier_window_function.sql diff --git a/tests/queries/0_stateless/03039_unknown_identifier_window_function.reference b/tests/queries/0_stateless/03039_unknown_identifier_window_function.reference new file mode 100644 index 00000000000..405da28a579 --- /dev/null +++ b/tests/queries/0_stateless/03039_unknown_identifier_window_function.reference @@ -0,0 +1,40 @@ +0 10 +1 10 +2 10 +3 10 +4 10 +5 10 +6 10 +7 10 +8 10 +9 10 +0 10 +1 10 +2 10 +3 10 +4 10 +5 10 +6 10 +7 10 +8 10 +9 10 +0 10 0 +1 10 1 +2 10 2 +3 10 3 +4 10 4 +5 10 5 +6 10 6 +7 10 7 +8 10 8 +9 10 9 +0 10 +1 10 +2 10 +3 10 +4 10 +5 10 +6 10 +7 10 +8 10 +9 10 diff --git a/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql b/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql new file mode 100644 index 00000000000..ca3bb521eba --- /dev/null +++ b/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql @@ -0,0 +1,34 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/45535 + +SELECT + *, + count() OVER () AS c +FROM numbers(10) +ORDER BY toString(number); + + +WITH + toString(number) as str +SELECT + *, + count() OVER () AS c +FROM numbers(10) +ORDER BY str; + +SELECT + *, + count() OVER () AS c, + toString(number) as str +FROM numbers(10) +ORDER BY str; + + +WITH + test AS ( + SELECT + *, + count() OVER () AS c + FROM numbers(10) + ) +SELECT * FROM test +ORDER BY toString(number); From 44847fbb2faf0d82a05c4b247d4540ca446f2269 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 2 Apr 2024 18:16:22 +0200 Subject: [PATCH 0841/1165] Analyzer: cover new analyzer with old analyzer fails --- .../03040_array_sum_and_join.reference | 5 ++++ .../0_stateless/03040_array_sum_and_join.sql | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 tests/queries/0_stateless/03040_array_sum_and_join.reference create mode 100644 tests/queries/0_stateless/03040_array_sum_and_join.sql diff --git a/tests/queries/0_stateless/03040_array_sum_and_join.reference b/tests/queries/0_stateless/03040_array_sum_and_join.reference new file mode 100644 index 00000000000..d81fd9a2f69 --- /dev/null +++ b/tests/queries/0_stateless/03040_array_sum_and_join.reference @@ -0,0 +1,5 @@ +79 name1 42.7027027027027 +62 name2 33.513513513513516 +44 name3 23.783783783783786 +[[1,2],[1,2]] +[(3,[1,2]),(4,[1,2])] diff --git a/tests/queries/0_stateless/03040_array_sum_and_join.sql b/tests/queries/0_stateless/03040_array_sum_and_join.sql new file mode 100644 index 00000000000..0084f0e4c7b --- /dev/null +++ b/tests/queries/0_stateless/03040_array_sum_and_join.sql @@ -0,0 +1,26 @@ + +select t.1 as cnt, + t.2 as name, + t.3 as percent +from ( + select arrayJoin(result) as t + from ( + select [ + (79, 'name1'), + (62, 'name2'), + (44, 'name3') + ] as data, + arraySum(arrayMap(t -> t.1, data)) as total, + arrayMap(t -> + tuple(t.1, t.2, + multiIf(total = 0, 0, t.1 > 0 and t.1 < 10, -1.0, + (toFloat32(t.1) / toFloat32(total)) * 100) + ), + data + ) as result + ) + ); + +SELECT arrayMap(x -> arrayMap(x -> (x.1), [(1, 1), (2, 2)]), [(3, 3), (4, 4)]); + +SELECT arrayMap(x -> (x.1, arrayMap(x -> (x.1), [(1, 1), (2, 2)])), [(3, 3), (4, 4)]); From a3028ed9cd4c88d8ab5bb86ab47a4ec1475df067 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 2 Apr 2024 16:20:09 +0000 Subject: [PATCH 0842/1165] Automatic style fix --- .../test_trace_collector_serverwide/test.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_trace_collector_serverwide/test.py b/tests/integration/test_trace_collector_serverwide/test.py index 5a7bba15fd7..88d235642b9 100644 --- a/tests/integration/test_trace_collector_serverwide/test.py +++ b/tests/integration/test_trace_collector_serverwide/test.py @@ -10,6 +10,7 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", main_configs=["configs/global_profiler.xml"]) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -19,8 +20,11 @@ def start_cluster(): finally: cluster.shutdown() + def test_global_thread_profiler(start_cluster): - node1.query("CREATE TABLE t (key UInt32, value String) Engine = MergeTree() ORDER BY key") + node1.query( + "CREATE TABLE t (key UInt32, value String) Engine = MergeTree() ORDER BY key" + ) node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") node1.query("INSERT INTO t SELECT number, toString(number) from numbers(100)") @@ -35,4 +39,11 @@ def test_global_thread_profiler(start_cluster): node1.query("SYSTEM FLUSH LOGS") - assert int(node1.query("SELECT count() FROM system.trace_log where trace_type='Real' and query_id = ''").strip()) > 0 + assert ( + int( + node1.query( + "SELECT count() FROM system.trace_log where trace_type='Real' and query_id = ''" + ).strip() + ) + > 0 + ) From 18402c6191dfa5bc4ec8a9278253f5bc241b60df Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 16:22:04 +0000 Subject: [PATCH 0843/1165] Close https://github.com/ClickHouse/ClickHouse/issues/44365 --- .../0_stateless/03040_alias_column_join.reference | 0 .../queries/0_stateless/03040_alias_column_join.sql | 13 +++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/queries/0_stateless/03040_alias_column_join.reference create mode 100644 tests/queries/0_stateless/03040_alias_column_join.sql diff --git a/tests/queries/0_stateless/03040_alias_column_join.reference b/tests/queries/0_stateless/03040_alias_column_join.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03040_alias_column_join.sql b/tests/queries/0_stateless/03040_alias_column_join.sql new file mode 100644 index 00000000000..f4ea2e5914d --- /dev/null +++ b/tests/queries/0_stateless/03040_alias_column_join.sql @@ -0,0 +1,13 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/44365 +DROP TABLE IF EXISTS 03040_test; + +CREATE TABLE 03040_test +( + id UInt64, + val String alias 'value: '||toString(id) +) ENGINE = MergeTree +ORDER BY tuple(); + +SELECT val FROM 03040_test t GROUP BY val; + +DROP TABLE IF EXISTS 03040_test; From 3e6168e63a117fe7c9c80346460bf8728645770e Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 2 Apr 2024 18:25:52 +0200 Subject: [PATCH 0844/1165] Exclude one more format string --- .../0_stateless/00002_log_and_exception_messages_formatting.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index b5710e29be9..74608360b9c 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -203,7 +203,7 @@ select with 0.16 as threshold select 'noisy Trace messages', - greatest(coalesce(((select message_format_string, count() from logs where level = 'Trace' and message_format_string not in ('Access granted: {}{}', '{} -> {}', 'Query {} to stage {}{}') + greatest(coalesce(((select message_format_string, count() from logs where level = 'Trace' and message_format_string not in ('Access granted: {}{}', '{} -> {}', 'Query {} to stage {}{}', 'Query {} from stage {} to stage {}{}') group by message_format_string order by count() desc limit 1) as top_message).2, 0) / (select count() from logs), threshold) as r, r <= threshold ? '' : top_message.1; From 170f50e095f9d6076c84ba8825c08310ebd55c8e Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 1 Apr 2024 09:57:10 +0200 Subject: [PATCH 0845/1165] More than 255 replicas in ReplicatedTableStatus --- src/Storages/MergeTree/ReplicatedTableStatus.h | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedTableStatus.h b/src/Storages/MergeTree/ReplicatedTableStatus.h index ce9ad3640f4..786a5fdb44d 100644 --- a/src/Storages/MergeTree/ReplicatedTableStatus.h +++ b/src/Storages/MergeTree/ReplicatedTableStatus.h @@ -24,8 +24,8 @@ struct ReplicatedTableStatus UInt64 log_max_index; UInt64 log_pointer; UInt64 absolute_delay; - UInt8 total_replicas; - UInt8 active_replicas; + UInt32 total_replicas; + UInt32 active_replicas; UInt64 lost_part_count; String last_queue_update_exception; /// If the error has happened fetching the info from ZooKeeper, this field will be set. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8ca061db4ec..b0f82e85ac4 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7003,7 +7003,7 @@ void StorageReplicatedMergeTree::getStatus(ReplicatedTableStatus & res, bool wit } res.log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); - res.total_replicas = all_replicas.size(); + res.total_replicas = UInt32(all_replicas.size()); if (get_result[1].error == Coordination::Error::ZNONODE) res.lost_part_count = 0; else From 478cabee22a66008988a0302c8b5111e59f6f70e Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 2 Apr 2024 18:33:56 +0200 Subject: [PATCH 0846/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/15411 --- .../03041_analyzer_gigachad_join.reference | 1 + .../0_stateless/03041_analyzer_gigachad_join.sql | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/queries/0_stateless/03041_analyzer_gigachad_join.reference create mode 100644 tests/queries/0_stateless/03041_analyzer_gigachad_join.sql diff --git a/tests/queries/0_stateless/03041_analyzer_gigachad_join.reference b/tests/queries/0_stateless/03041_analyzer_gigachad_join.reference new file mode 100644 index 00000000000..a859a600512 --- /dev/null +++ b/tests/queries/0_stateless/03041_analyzer_gigachad_join.reference @@ -0,0 +1 @@ +123456789 111 222 diff --git a/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql b/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql new file mode 100644 index 00000000000..462e63b121b --- /dev/null +++ b/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS first engine = MergeTree PARTITION BY (inn, toYYYYMM(received)) ORDER BY (inn, sessionId) +AS SELECT now() AS received, '123456789' AS inn, '42' AS sessionId; + +CREATE TABLE IF NOT EXISTS second engine = MergeTree PARTITION BY (inn, toYYYYMM(received)) ORDER BY (inn, sessionId) +AS SELECT now() AS received, '123456789' AS inn, '42' AS sessionId, '111' AS serial, '222' AS reg; + +SELECT alias_first.inn, arrayFirst(t -> isNotNull(t), regInfo.1), arrayFirst(t -> isNotNull(t), regInfo.2) + FROM first AS alias_first + INNER JOIN ( + SELECT alias_second.inn, alias_second.sessionId, groupArray((serial, reg)) AS regInfo + FROM second AS alias_second + GROUP BY inn, sessionId + ) AS resp ON (alias_first.inn = resp.inn) AND (alias_first.sessionId = resp.sessionId) +WHERE if('123456789' IS NOT NULL, alias_first.inn = '123456789', 1) From a4a56a9b6c0ce7ce66006be87d8bf0ef8b660aed Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 16:38:18 +0000 Subject: [PATCH 0847/1165] Close https://github.com/ClickHouse/ClickHouse/issues/44153 --- .../03041_select_with_query_result.reference | 0 .../03041_select_with_query_result.sql | 41 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/queries/0_stateless/03041_select_with_query_result.reference create mode 100644 tests/queries/0_stateless/03041_select_with_query_result.sql diff --git a/tests/queries/0_stateless/03041_select_with_query_result.reference b/tests/queries/0_stateless/03041_select_with_query_result.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03041_select_with_query_result.sql b/tests/queries/0_stateless/03041_select_with_query_result.sql new file mode 100644 index 00000000000..3edf51d635e --- /dev/null +++ b/tests/queries/0_stateless/03041_select_with_query_result.sql @@ -0,0 +1,41 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/44153 +DROP TABLE IF EXISTS parent; +DROP TABLE IF EXISTS join_table_1; +DROP TABLE IF EXISTS join_table_2; + +CREATE TABLE parent( + a_id Int64, + b_id Int64, + c_id Int64, + created_at Int64 +) +ENGINE=MergeTree() +ORDER BY (a_id, b_id, c_id, created_at); + +CREATE TABLE join_table_1( + a_id Int64, + b_id Int64 +) +ENGINE=MergeTree() +ORDER BY (a_id, b_id); + +CREATE TABLE join_table_2( + c_id Int64, + created_at Int64 +) +ENGINE=MergeTree() +ORDER BY (c_id, created_at); + +WITH with_table as ( + SELECT p.a_id, p.b_id, p.c_id FROM parent p + LEFT JOIN join_table_1 jt1 ON jt1.a_id = p.a_id AND jt1.b_id = p.b_id + LEFT JOIN join_table_2 jt2 ON jt2.c_id = p.c_id + WHERE + p.a_id = 0 AND (jt2.c_id = 0 OR p.created_at = 0) +) +SELECT p.a_id, p.b_id, COUNT(*) as f_count FROM with_table +GROUP BY p.a_id, p.b_id; + +DROP TABLE IF EXISTS parent; +DROP TABLE IF EXISTS join_table_1; +DROP TABLE IF EXISTS join_table_2; From 0a5747377bb44bee51dcf6223930c16529cb9a83 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 2 Apr 2024 18:43:12 +0200 Subject: [PATCH 0848/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/14978 --- .../03042_analyzer_alias_join.reference | 0 .../0_stateless/03042_analyzer_alias_join.sql | 20 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/queries/0_stateless/03042_analyzer_alias_join.reference create mode 100644 tests/queries/0_stateless/03042_analyzer_alias_join.sql diff --git a/tests/queries/0_stateless/03042_analyzer_alias_join.reference b/tests/queries/0_stateless/03042_analyzer_alias_join.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03042_analyzer_alias_join.sql b/tests/queries/0_stateless/03042_analyzer_alias_join.sql new file mode 100644 index 00000000000..f3341fd314a --- /dev/null +++ b/tests/queries/0_stateless/03042_analyzer_alias_join.sql @@ -0,0 +1,20 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/14978 +CREATE TABLE test1(id UInt64, t1value UInt64) ENGINE=MergeTree ORDER BY tuple(); +CREATE TABLE test2(id UInt64, t2value String) ENGINE=MergeTree ORDER BY tuple(); + +SELECT NULL AS t2value +FROM test1 t1 +LEFT JOIN ( + SELECT id, t2value FROM test2 +) t2 +ON t1.id=t2.id +WHERE t2.t2value='test'; + +-- workaround should work too +SELECT NULL AS _svalue +FROM test1 t1 +LEFT JOIN ( + SELECT id, t2value FROM test2 +) t2 +ON t1.id=t2.id +WHERE t2.t2value='test'; From 1fb23c64f15b70228f7b0911f4e5358c4a077b61 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 16:41:42 +0000 Subject: [PATCH 0849/1165] Close https://github.com/ClickHouse/ClickHouse/issues/42399 --- .../0_stateless/03042_not_found_column_c1.reference | 0 tests/queries/0_stateless/03042_not_found_column_c1.sql | 8 ++++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/03042_not_found_column_c1.reference create mode 100644 tests/queries/0_stateless/03042_not_found_column_c1.sql diff --git a/tests/queries/0_stateless/03042_not_found_column_c1.reference b/tests/queries/0_stateless/03042_not_found_column_c1.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03042_not_found_column_c1.sql b/tests/queries/0_stateless/03042_not_found_column_c1.sql new file mode 100644 index 00000000000..8ce7dcd9d4f --- /dev/null +++ b/tests/queries/0_stateless/03042_not_found_column_c1.sql @@ -0,0 +1,8 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/42399 + +CREATE TABLE IF NOT EXISTS t0 (c0 Int32) ENGINE = Memory() ; +CREATE TABLE t1 (c0 Int32, c1 Int32, c2 Int32) ENGINE = Memory() ; +CREATE TABLE t2 (c0 String, c1 String MATERIALIZED (c2), c2 Int32) ENGINE = Memory() ; +CREATE TABLE t3 (c0 String, c1 String, c2 String) ENGINE = Log() ; +CREATE TABLE IF NOT EXISTS t4 (c0 Int32) ENGINE = Log() ; +SELECT t3.c1, t3.c2, t1.c1, t1.c0, t2.c2, t0.c0, t1.c2, t2.c1, t4.c0 FROM t3, t0, t1, t2, t4; From 2d8f07318c06ec330c8d6e87facd387bc2b63341 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 16:45:21 +0000 Subject: [PATCH 0850/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/27115 --- ...3_group_array_result_is_expected.reference | 1 + .../03043_group_array_result_is_expected.sql | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/queries/0_stateless/03043_group_array_result_is_expected.reference create mode 100644 tests/queries/0_stateless/03043_group_array_result_is_expected.sql diff --git a/tests/queries/0_stateless/03043_group_array_result_is_expected.reference b/tests/queries/0_stateless/03043_group_array_result_is_expected.reference new file mode 100644 index 00000000000..d43aa556dce --- /dev/null +++ b/tests/queries/0_stateless/03043_group_array_result_is_expected.reference @@ -0,0 +1 @@ +['2021-07-01','2021-07-02','2021-07-03','2021-07-04','2021-07-05','2021-07-06','2021-07-07','2021-07-08','2021-07-09','2021-07-10','2021-07-11','2021-07-12','2021-07-13','2021-07-14','2021-07-15','2021-07-16','2021-07-17','2021-07-18','2021-07-19','2021-07-20','2021-07-21','2021-07-22','2021-07-23','2021-07-24','2021-07-25','2021-07-26','2021-07-27','2021-07-28','2021-07-29'] 29 diff --git a/tests/queries/0_stateless/03043_group_array_result_is_expected.sql b/tests/queries/0_stateless/03043_group_array_result_is_expected.sql new file mode 100644 index 00000000000..df77ca66647 --- /dev/null +++ b/tests/queries/0_stateless/03043_group_array_result_is_expected.sql @@ -0,0 +1,44 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/27115 +drop table if exists fill_ex; + +create table fill_ex ( + eventDate Date , + storeId String +) +engine = ReplacingMergeTree() +partition by toYYYYMM(eventDate) +order by (storeId,eventDate); + +insert into fill_ex (eventDate,storeId) values ('2021-07-16','s') ('2021-07-17','ee'); + +select + groupArray(key) as keys, + count() as c +from + ( + select + *, + eventDate as key + from + ( + select + eventDate + from + ( + select + eventDate + from + fill_ex final + where + eventDate >= toDate('2021-07-01') + and eventDate Date: Tue, 2 Apr 2024 17:48:28 +0100 Subject: [PATCH 0851/1165] Bump From ed0522ae1a2e96e59386a7ed25ba85a774d429ad Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 2 Apr 2024 18:48:00 +0200 Subject: [PATCH 0852/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/17319 --- .../03044_analyzer_alias_join.reference | 0 .../0_stateless/03044_analyzer_alias_join.sql | 17 +++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/03044_analyzer_alias_join.reference create mode 100644 tests/queries/0_stateless/03044_analyzer_alias_join.sql diff --git a/tests/queries/0_stateless/03044_analyzer_alias_join.reference b/tests/queries/0_stateless/03044_analyzer_alias_join.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03044_analyzer_alias_join.sql b/tests/queries/0_stateless/03044_analyzer_alias_join.sql new file mode 100644 index 00000000000..5202b57a7b1 --- /dev/null +++ b/tests/queries/0_stateless/03044_analyzer_alias_join.sql @@ -0,0 +1,17 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/17319 +CREATE TEMPORARY TABLE hits (date Date, data Float64) engine=Memory(); + +SELECT + subquery1.period AS period, + if(1=1, 0, subquery1.data1) AS data, + if(1=1, 0, subquery2.data) AS other_data +FROM +( + SELECT date AS period, data AS data1 + FROM hits +) AS subquery1 +LEFT JOIN +( + SELECT date AS period, data AS data + FROM hits +) AS subquery2 ON (subquery1.period = subquery2.period) From f1fb042be3d54d0347568abcc6e5c3358665d075 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 2 Apr 2024 18:55:25 +0200 Subject: [PATCH 0853/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/13210 --- ...3045_analyzer_alias_join_with_if.reference | 0 .../03045_analyzer_alias_join_with_if.sql | 33 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 tests/queries/0_stateless/03045_analyzer_alias_join_with_if.reference create mode 100644 tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql diff --git a/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.reference b/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql b/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql new file mode 100644 index 00000000000..a0546f57736 --- /dev/null +++ b/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql @@ -0,0 +1,33 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/13210 +CREATE TABLE test_a_table ( + name String, + a_col String +) +Engine = MergeTree() +ORDER BY name; + +CREATE TABLE test_b_table ( + name String, + b_col String, + some_val String +) +Engine = MergeTree() +ORDER BY name; + +SELECT + b.name name, + a.a_col a_col, + b.b_col b_col, + 'N' some_val +from test_a_table a +join test_b_table b on a.name = b.name +where b.some_val = 'Y'; + +SELECT + b.name name, + a.a_col a_col, + b.b_col b_col, + if(1,'N',b.some_val) some_val +from test_a_table a +join test_b_table b on a.name = b.name +where b.some_val = 'Y'; From b19b0890becd3cf9f3d7e23744e0e477ca12e4ef Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 16:56:16 +0000 Subject: [PATCH 0854/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/11813 --- .../03044_array_join_columns_in_nested_table.reference | 1 + .../0_stateless/03044_array_join_columns_in_nested_table.sql | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 tests/queries/0_stateless/03044_array_join_columns_in_nested_table.reference create mode 100644 tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql diff --git a/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.reference b/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql b/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql new file mode 100644 index 00000000000..f3ec80b8a94 --- /dev/null +++ b/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql @@ -0,0 +1,2 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/11813 +select 1 from (select 1 x) l join (select 1 y, [1] a) r on l.x = r.y array join r.a; From 042e612485d7083749342b8658bbb5e580da580a Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 16:59:03 +0000 Subject: [PATCH 0855/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/23053 --- ...wn_identifier_alias_substitution.reference | 0 ..._unknown_identifier_alias_substitution.sql | 20 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.reference create mode 100644 tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql diff --git a/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.reference b/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql b/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql new file mode 100644 index 00000000000..cadcbdc0ce5 --- /dev/null +++ b/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql @@ -0,0 +1,20 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/23053 +DROP TABLE IF EXISTS repl_tbl; + +CREATE TEMPORARY TABLE repl_tbl +( + `key` UInt32, + `val_1` UInt32, + `val_2` String, + `val_3` String, + `val_4` String, + `val_5` UUID, + `ts` DateTime +) +ENGINE = ReplacingMergeTree(ts) +ORDER BY `key`; +set prefer_column_name_to_alias = 1; +INSERT INTO repl_tbl (key) SELECT number FROM numbers(10); +WITH 10 as k SELECT k as key, * FROM repl_tbl WHERE key = k; + +DROP TABLE IF EXISTS repl_tbl; From 668aa9bafd25b6c27a8aba02dd0b1d53c782fc65 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 17:03:40 +0000 Subject: [PATCH 0856/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/37729 --- ...03046_column_in_block_array_join.reference | 2 + .../03046_column_in_block_array_join.sql | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/queries/0_stateless/03046_column_in_block_array_join.reference create mode 100644 tests/queries/0_stateless/03046_column_in_block_array_join.sql diff --git a/tests/queries/0_stateless/03046_column_in_block_array_join.reference b/tests/queries/0_stateless/03046_column_in_block_array_join.reference new file mode 100644 index 00000000000..f9264f7fbd3 --- /dev/null +++ b/tests/queries/0_stateless/03046_column_in_block_array_join.reference @@ -0,0 +1,2 @@ +Hello +World diff --git a/tests/queries/0_stateless/03046_column_in_block_array_join.sql b/tests/queries/0_stateless/03046_column_in_block_array_join.sql new file mode 100644 index 00000000000..9a2bb19d81e --- /dev/null +++ b/tests/queries/0_stateless/03046_column_in_block_array_join.sql @@ -0,0 +1,37 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/37729 + +DROP TABLE IF EXISTS nested_test; +DROP TABLE IF EXISTS join_test; + +CREATE TABLE nested_test +( + s String, + nest Nested + ( + x UInt64, + y UInt64 + ) +) ENGINE = MergeTree +ORDER BY s; + +CREATE TABLE join_test +( + id Int64, + y UInt64 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO nested_test +VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); + +INSERT INTO join_test +VALUES (1,1),(2,4),(3,20),(4,40); + +SELECT s +FROM nested_test AS t1 +ARRAY JOIN nest +INNER JOIN join_test AS t2 ON nest.y = t2.y; + +DROP TABLE IF EXISTS nested_test; +DROP TABLE IF EXISTS join_test; From 9b74e246af4eec622ca749aebbbe2985428e51f7 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 2 Apr 2024 19:03:54 +0200 Subject: [PATCH 0857/1165] Rename a method --- src/Interpreters/Cache/FileSegment.cpp | 132 ++++++++++++------------- src/Interpreters/Cache/FileSegment.h | 6 +- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 0d64b602928..f56f5d3f66c 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -113,7 +113,7 @@ FileSegment::Range::Range(size_t left_, size_t right_) : left(left_), right(righ FileSegment::State FileSegment::state() const { - auto lock = lockFileSegment(); + auto lk = lock(); return download_state; } @@ -130,7 +130,7 @@ String FileSegment::tryGetPath() const return metadata->getFileSegmentPath(*this); } -FileSegmentGuard::Lock FileSegment::lockFileSegment() const +FileSegmentGuard::Lock FileSegment::lock() const { ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentLockMicroseconds); return segment_guard.lock(); @@ -152,29 +152,30 @@ void FileSegment::setDownloadState(State state, const FileSegmentGuard::Lock & l size_t FileSegment::getReservedSize() const { - auto lock = lockFileSegment(); + auto lk = lock(); return reserved_size; } FileSegment::Priority::IteratorPtr FileSegment::getQueueIterator() const { - auto lock = lockFileSegment(); + auto lk = lock(); return queue_iterator; } void FileSegment::setQueueIterator(Priority::IteratorPtr iterator) { - auto lock = lockFileSegment(); - setQueueIteratorUnlocked(iterator, lock); -} - -void FileSegment::setQueueIteratorUnlocked(Priority::IteratorPtr iterator, const FileSegmentGuard::Lock &) -{ + auto lk = lock(); if (queue_iterator) throw Exception(ErrorCodes::LOGICAL_ERROR, "Queue iterator cannot be set twice"); queue_iterator = iterator; } +void FileSegment::resetQueueIterator() +{ + auto lk = lock(); + queue_iterator.reset(); +} + size_t FileSegment::getCurrentWriteOffset() const { return range().left + downloaded_size; @@ -187,14 +188,14 @@ size_t FileSegment::getDownloadedSize() const void FileSegment::setDownloadedSize(size_t delta) { - auto lock = lockFileSegment(); + auto lk = lock(); downloaded_size += delta; assert(downloaded_size == std::filesystem::file_size(getPath())); } bool FileSegment::isDownloaded() const { - auto lock = lockFileSegment(); + auto lk = lock(); return download_state == State::DOWNLOADED; } @@ -208,8 +209,7 @@ String FileSegment::getCallerId() String FileSegment::getDownloader() const { - auto lock = lockFileSegment(); - return getDownloaderUnlocked(lock); + return getDownloaderUnlocked(lock()); } String FileSegment::getDownloaderUnlocked(const FileSegmentGuard::Lock &) const @@ -219,11 +219,11 @@ String FileSegment::getDownloaderUnlocked(const FileSegmentGuard::Lock &) const String FileSegment::getOrSetDownloader() { - auto lock = lockFileSegment(); + auto lk = lock(); - assertNotDetachedUnlocked(lock); + assertNotDetachedUnlocked(lk); - auto current_downloader = getDownloaderUnlocked(lock); + auto current_downloader = getDownloaderUnlocked(lk); if (current_downloader.empty()) { @@ -233,7 +233,7 @@ String FileSegment::getOrSetDownloader() return "notAllowed:" + stateToString(download_state); current_downloader = downloader_id = caller_id; - setDownloadState(State::DOWNLOADING, lock); + setDownloadState(State::DOWNLOADING, lk); chassert(key_metadata.lock()); } @@ -257,15 +257,15 @@ void FileSegment::resetDownloadingStateUnlocked(const FileSegmentGuard::Lock & l void FileSegment::resetDownloader() { - auto lock = lockFileSegment(); + auto lk = lock(); SCOPE_EXIT({ cv.notify_all(); }); - assertNotDetachedUnlocked(lock); - assertIsDownloaderUnlocked("resetDownloader", lock); + assertNotDetachedUnlocked(lk); + assertIsDownloaderUnlocked("resetDownloader", lk); - resetDownloadingStateUnlocked(lock); - resetDownloaderUnlocked(lock); + resetDownloadingStateUnlocked(lk); + resetDownloaderUnlocked(lk); } void FileSegment::resetDownloaderUnlocked(const FileSegmentGuard::Lock &) @@ -294,8 +294,8 @@ void FileSegment::assertIsDownloaderUnlocked(const std::string & operation, cons bool FileSegment::isDownloader() const { - auto lock = lockFileSegment(); - return isDownloaderUnlocked(lock); + auto lk = lock(); + return isDownloaderUnlocked(lk); } bool FileSegment::isDownloaderUnlocked(const FileSegmentGuard::Lock & lock) const @@ -305,21 +305,21 @@ bool FileSegment::isDownloaderUnlocked(const FileSegmentGuard::Lock & lock) cons FileSegment::RemoteFileReaderPtr FileSegment::getRemoteFileReader() { - auto lock = lockFileSegment(); - assertIsDownloaderUnlocked("getRemoteFileReader", lock); + auto lk = lock(); + assertIsDownloaderUnlocked("getRemoteFileReader", lk); return remote_file_reader; } void FileSegment::resetRemoteFileReader() { - auto lock = lockFileSegment(); - assertIsDownloaderUnlocked("resetRemoteFileReader", lock); + auto lk = lock(); + assertIsDownloaderUnlocked("resetRemoteFileReader", lk); remote_file_reader.reset(); } FileSegment::RemoteFileReaderPtr FileSegment::extractRemoteFileReader() { - auto lock = lockFileSegment(); + auto lk = lock(); if (remote_file_reader && (download_state == State::DOWNLOADED || download_state == State::PARTIALLY_DOWNLOADED_NO_CONTINUATION)) { @@ -330,8 +330,8 @@ FileSegment::RemoteFileReaderPtr FileSegment::extractRemoteFileReader() void FileSegment::setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_) { - auto lock = lockFileSegment(); - assertIsDownloaderUnlocked("setRemoteFileReader", lock); + auto lk = lock(); + assertIsDownloaderUnlocked("setRemoteFileReader", lk); if (remote_file_reader) throw Exception(ErrorCodes::LOGICAL_ERROR, "Remote file reader already exists"); @@ -347,9 +347,9 @@ void FileSegment::write(const char * from, size_t size, size_t offset) throw Exception(ErrorCodes::LOGICAL_ERROR, "Writing zero size is not allowed"); { - auto lock = lockFileSegment(); - assertIsDownloaderUnlocked("write", lock); - assertNotDetachedUnlocked(lock); + auto lk = lock(); + assertIsDownloaderUnlocked("write", lk); + assertNotDetachedUnlocked(lk); } const auto file_segment_path = getPath(); @@ -408,10 +408,10 @@ void FileSegment::write(const char * from, size_t size, size_t offset) const int code = e.getErrno(); const bool is_no_space_left_error = code == /* No space left on device */28 || code == /* Quota exceeded */122; - auto lock = lockFileSegment(); + auto lk = lock(); - e.addMessage(fmt::format("{}, current cache state: {}", e.what(), getInfoForLogUnlocked(lock))); - setDownloadFailedUnlocked(lock); + e.addMessage(fmt::format("{}, current cache state: {}", e.what(), getInfoForLogUnlocked(lk))); + setDownloadFailedUnlocked(lk); if (downloaded_size == 0 && fs::exists(file_segment_path)) { @@ -434,9 +434,9 @@ void FileSegment::write(const char * from, size_t size, size_t offset) } catch (Exception & e) { - auto lock = lockFileSegment(); - e.addMessage(fmt::format("{}, current cache state: {}", e.what(), getInfoForLogUnlocked(lock))); - setDownloadFailedUnlocked(lock); + auto lk = lock(); + e.addMessage(fmt::format("{}, current cache state: {}", e.what(), getInfoForLogUnlocked(lk))); + setDownloadFailedUnlocked(lk); throw; } @@ -449,7 +449,7 @@ FileSegment::State FileSegment::wait(size_t offset) span.addAttribute("clickhouse.key", key().toString()); span.addAttribute("clickhouse.offset", offset); - auto lock = lockFileSegment(); + auto lk = lock(); if (downloader_id.empty() || offset < getCurrentWriteOffset()) return download_state; @@ -462,10 +462,10 @@ FileSegment::State FileSegment::wait(size_t offset) LOG_TEST(log, "{} waiting on: {}, current downloader: {}", getCallerId(), range().toString(), downloader_id); ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentWaitMicroseconds); - chassert(!getDownloaderUnlocked(lock).empty()); - chassert(!isDownloaderUnlocked(lock)); + chassert(!getDownloaderUnlocked(lk).empty()); + chassert(!isDownloaderUnlocked(lk)); - [[maybe_unused]] const auto ok = cv.wait_for(lock, std::chrono::seconds(60), [&, this]() + [[maybe_unused]] const auto ok = cv.wait_for(lk, std::chrono::seconds(60), [&, this]() { return download_state != State::DOWNLOADING || offset < getCurrentWriteOffset(); }); @@ -511,10 +511,10 @@ bool FileSegment::reserve(size_t size_to_reserve, size_t lock_wait_timeout_milli bool is_file_segment_size_exceeded; { - auto lock = lockFileSegment(); + auto lk = lock(); - assertNotDetachedUnlocked(lock); - assertIsDownloaderUnlocked("reserve", lock); + assertNotDetachedUnlocked(lk); + assertIsDownloaderUnlocked("reserve", lk); expected_downloaded_size = getDownloadedSize(); @@ -557,7 +557,7 @@ bool FileSegment::reserve(size_t size_to_reserve, size_t lock_wait_timeout_milli bool reserved = cache->tryReserve(*this, size_to_reserve, *reserve_stat, getKeyMetadata()->user, lock_wait_timeout_milliseconds); if (!reserved) - setDownloadFailedUnlocked(lockFileSegment()); + setDownloadFailedUnlocked(lock()); return reserved; } @@ -582,8 +582,8 @@ void FileSegment::setDownloadedUnlocked(const FileSegmentGuard::Lock &) void FileSegment::setDownloadFailed() { - auto lock = lockFileSegment(); - setDownloadFailedUnlocked(lock); + auto lk = lock(); + setDownloadFailedUnlocked(lk); } void FileSegment::setDownloadFailedUnlocked(const FileSegmentGuard::Lock & lock) @@ -605,22 +605,22 @@ void FileSegment::setDownloadFailedUnlocked(const FileSegmentGuard::Lock & lock) void FileSegment::completePartAndResetDownloader() { - auto lock = lockFileSegment(); + auto lk = lock(); SCOPE_EXIT({ cv.notify_all(); }); - assertNotDetachedUnlocked(lock); - assertIsDownloaderUnlocked("completePartAndResetDownloader", lock); + assertNotDetachedUnlocked(lk); + assertIsDownloaderUnlocked("completePartAndResetDownloader", lk); chassert(download_state == State::DOWNLOADING || download_state == State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); if (download_state == State::DOWNLOADING) - resetDownloadingStateUnlocked(lock); + resetDownloadingStateUnlocked(lk); - resetDownloaderUnlocked(lock); + resetDownloaderUnlocked(lk); - LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lock)); + LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lk)); } void FileSegment::complete() @@ -640,7 +640,7 @@ void FileSegment::complete() throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot complete file segment: {}", getInfoForLog()); } - auto segment_lock = lockFileSegment(); + auto segment_lock = lock(); if (isCompleted(false)) return; @@ -756,8 +756,8 @@ void FileSegment::complete() String FileSegment::getInfoForLog() const { - auto lock = lockFileSegment(); - return getInfoForLogUnlocked(lock); + auto lk = lock(); + return getInfoForLogUnlocked(lk); } String FileSegment::getInfoForLogUnlocked(const FileSegmentGuard::Lock &) const @@ -799,7 +799,7 @@ String FileSegment::stateToString(FileSegment::State state) bool FileSegment::assertCorrectness() const { - return assertCorrectnessUnlocked(lockFileSegment()); + return assertCorrectnessUnlocked(lock()); } bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) const @@ -869,8 +869,8 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) void FileSegment::assertNotDetached() const { - auto lock = lockFileSegment(); - assertNotDetachedUnlocked(lock); + auto lk = lock(); + assertNotDetachedUnlocked(lk); } void FileSegment::assertNotDetachedUnlocked(const FileSegmentGuard::Lock & lock) const @@ -887,7 +887,7 @@ void FileSegment::assertNotDetachedUnlocked(const FileSegmentGuard::Lock & lock) FileSegment::Info FileSegment::getInfo(const FileSegmentPtr & file_segment) { - auto lock = file_segment->lockFileSegment(); + auto lock = file_segment->lock(); auto key_metadata = file_segment->tryGetKeyMetadata(); return Info{ .key = file_segment->key(), @@ -910,7 +910,7 @@ FileSegment::Info FileSegment::getInfo(const FileSegmentPtr & file_segment) bool FileSegment::isDetached() const { - auto lock = lockFileSegment(); + auto lk = lock(); return download_state == State::DETACHED; } @@ -926,7 +926,7 @@ bool FileSegment::isCompleted(bool sync) const if (is_completed_state()) return true; - auto lock = lockFileSegment(); + auto lk = lock(); return is_completed_state(); } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 12046e59bd6..7793c50d2d5 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -171,12 +171,13 @@ public: * ========== Methods used by `cache` ======================== */ - FileSegmentGuard::Lock lock() const { return segment_guard.lock(); } + FileSegmentGuard::Lock lock() const; Priority::IteratorPtr getQueueIterator() const; void setQueueIterator(Priority::IteratorPtr iterator); - void setQueueIteratorUnlocked(Priority::IteratorPtr iterator, const FileSegmentGuard::Lock &); + + void resetQueueIterator(); KeyMetadataPtr tryGetKeyMetadata() const; @@ -242,7 +243,6 @@ private: bool assertCorrectnessUnlocked(const FileSegmentGuard::Lock &) const; LockedKeyPtr lockKeyMetadata(bool assert_exists = true) const; - FileSegmentGuard::Lock lockFileSegment() const; String tryGetPath() const; From 607a9d761babcae14c9d4ff64c8753900464e2b8 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 2 Apr 2024 19:04:02 +0200 Subject: [PATCH 0858/1165] Fix --- src/Interpreters/Cache/EvictionCandidates.cpp | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 4ca6aeea22e..5fa2b337e64 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -67,11 +67,11 @@ void EvictionCandidates::removeQueueEntries(const CachePriorityGuard::Lock & loc { for (const auto & candidate : key_candidates.candidates) { - const auto & file_segment = candidate->file_segment; - auto file_segment_lock = file_segment->lock(); + auto queue_iterator = candidate->getQueueIterator(); + queue_iterator->invalidate(); - candidate->getQueueIterator()->remove(lock); - file_segment->setQueueIteratorUnlocked(nullptr, file_segment_lock); + candidate->file_segment->resetQueueIterator(); + queue_iterator->remove(lock); } } removed_queue_entries = true; @@ -101,10 +101,14 @@ void EvictionCandidates::evict() { auto & candidate = key_candidates.candidates.back(); chassert(candidate->releasable()); - const auto segment = candidate->file_segment; - auto iterator = segment->getQueueIterator(); - chassert(iterator); + + IFileCachePriority::IteratorPtr iterator; + if (!removed_queue_entries) + { + iterator = segment->getQueueIterator(); + chassert(iterator); + } ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedFileSegments); ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedBytes, segment->range().size()); @@ -133,7 +137,7 @@ void EvictionCandidates::evict() /// it was freed in favour of some reserver, so we can make it visibly /// free only for that particular reserver. - if (!removed_queue_entries) + if (iterator) queue_entries_to_invalidate.push_back(iterator); key_candidates.candidates.pop_back(); From 2fd23a15653cef89400a914824e44a27c823beeb Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 2 Apr 2024 19:05:39 +0200 Subject: [PATCH 0859/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/9954 --- .../03047_analyzer_alias_join.reference | 2 ++ .../0_stateless/03047_analyzer_alias_join.sql | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/queries/0_stateless/03047_analyzer_alias_join.reference create mode 100644 tests/queries/0_stateless/03047_analyzer_alias_join.sql diff --git a/tests/queries/0_stateless/03047_analyzer_alias_join.reference b/tests/queries/0_stateless/03047_analyzer_alias_join.reference new file mode 100644 index 00000000000..3bb92083849 --- /dev/null +++ b/tests/queries/0_stateless/03047_analyzer_alias_join.reference @@ -0,0 +1,2 @@ +1 1 \N +1 2 1 diff --git a/tests/queries/0_stateless/03047_analyzer_alias_join.sql b/tests/queries/0_stateless/03047_analyzer_alias_join.sql new file mode 100644 index 00000000000..a6f81516430 --- /dev/null +++ b/tests/queries/0_stateless/03047_analyzer_alias_join.sql @@ -0,0 +1,30 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/9954 +SELECT + 1 AS value, + * +FROM +( + SELECT 1 AS key +) AS l +LEFT JOIN +( + SELECT + 2 AS key, + 1 AS value +) AS r USING (key) +SETTINGS join_use_nulls = 1; + +SELECT + 1 AS value, + * +FROM +( + SELECT 2 AS key +) AS l +LEFT JOIN +( + SELECT + 2 AS key, + 1 AS value +) AS r USING (key) +SETTINGS join_use_nulls = 1 From a19bce649aef3798972e8fe4129bbc0ac4dbdd84 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 17:07:55 +0000 Subject: [PATCH 0860/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/32639 --- .../03047_group_by_field_identified_aggregation.reference | 1 + .../03047_group_by_field_identified_aggregation.sql | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 tests/queries/0_stateless/03047_group_by_field_identified_aggregation.reference create mode 100644 tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql diff --git a/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.reference b/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.reference new file mode 100644 index 00000000000..556d825db42 --- /dev/null +++ b/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.reference @@ -0,0 +1 @@ +2 1 diff --git a/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql b/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql new file mode 100644 index 00000000000..e1363ea4dda --- /dev/null +++ b/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql @@ -0,0 +1,3 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/32639 + +SELECT 0 AND id ? 1 : 2 AS a, sum(id) FROM (SELECT 1 AS id) GROUP BY a; From a3f973662289a7ad218f8bf7994b207b4891e64f Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 17:13:13 +0000 Subject: [PATCH 0861/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/41964 --- ...48_not_found_column_xxx_in_block.reference | 0 .../03048_not_found_column_xxx_in_block.sql | 66 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 tests/queries/0_stateless/03048_not_found_column_xxx_in_block.reference create mode 100644 tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql diff --git a/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.reference b/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql b/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql new file mode 100644 index 00000000000..25f88050eb1 --- /dev/null +++ b/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql @@ -0,0 +1,66 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/41964 + +DROP TABLE IF EXISTS ab_12_aaa; +DROP TABLE IF EXISTS ab_12_bbb; + +CREATE TABLE ab_12_aaa +( + `id` String, + `subid` Int32, + `prodcat` String, + `prodtype` String, + `quality` String, + `m1` Float64, + `m2` Float64, + `r1` Float64, + `r2` Float64, + `d1` Float64, + `d2` Float64, + `pcs` Float64, + `qty` Float64, + `amt` Float64, + `amts` Float64, + `prc` Float64, + `prcs` Float64, + `suqty` Float64, + `suamt` Float64, + `_year` String +) +ENGINE = MergeTree +ORDER BY (_year, prodcat, prodtype, quality, d1, id) +SETTINGS index_granularity = 8192; + +CREATE TABLE ab_12_bbb +( + `id` String, + `sales_type` String, + `date` Date32, + `o1` String, + `o2` String, + `o3` String, + `o4` String, + `o5` String, + `short` String, + `a1` String, + `a2` String, + `a3` String, + `idx` String, + `a4` String, + `ctx` String, + `_year` String, + `theyear` UInt16 MATERIALIZED toYear(`date`), + `themonth` UInt8 MATERIALIZED toMonth(`date`), + `theweek` UInt8 MATERIALIZED toISOWeek(`date`) +) +ENGINE = MergeTree +ORDER BY (theyear, themonth, _year, id, sales_type, date) +SETTINGS index_granularity = 8192; + +SELECT * +FROM ab_12_aaa aa +LEFT JOIN ab_12_bbb bb +ON bb.id = aa.id AND bb.`_year` = aa.`_year` +WHERE bb.theyear >= 2019; + +DROP TABLE IF EXISTS ab_12_aaa; +DROP TABLE IF EXISTS ab_12_bbb; From 14820032799b252091910a3fd999866078976761 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 2 Apr 2024 19:16:49 +0200 Subject: [PATCH 0862/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/7520 --- .../03049_analyzer_group_by_alias.reference | 7 ++++++ .../03049_analyzer_group_by_alias.sql | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/queries/0_stateless/03049_analyzer_group_by_alias.reference create mode 100644 tests/queries/0_stateless/03049_analyzer_group_by_alias.sql diff --git a/tests/queries/0_stateless/03049_analyzer_group_by_alias.reference b/tests/queries/0_stateless/03049_analyzer_group_by_alias.reference new file mode 100644 index 00000000000..dac5487d445 --- /dev/null +++ b/tests/queries/0_stateless/03049_analyzer_group_by_alias.reference @@ -0,0 +1,7 @@ +1 5 +2 4 +1 +2 +1 +1 +2 diff --git a/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql b/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql new file mode 100644 index 00000000000..67df48e0cad --- /dev/null +++ b/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql @@ -0,0 +1,22 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/7520 +CREATE TABLE test (`a` UInt32, `b` UInt32) ENGINE = Memory; + +INSERT INTO test VALUES (1,2), (1,3), (2,4); + +-- 1 5 +-- 2 4 + +WITH + a as key +SELECT + a as k1, + sum(b) as k2 +FROM + test +GROUP BY + key +ORDER BY k1, k2; + +WITH a as key SELECT key as k1 FROM test GROUP BY key ORDER BY key; + +WITH a as key SELECT key as k1 FROM test ORDER BY key; From 167c993503f62ea2e9b91816a17e25ae48aaf98d Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 17:17:10 +0000 Subject: [PATCH 0863/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/54317 --- ...9_unknown_identifier_materialized_column.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql diff --git a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql new file mode 100644 index 00000000000..2f12799addb --- /dev/null +++ b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql @@ -0,0 +1,17 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/54317 +DROP DATABASE IF EXISTS 03049_database; +DROP TABLE IF EXISTS 03049_database.l; +DROP TABLE IF EXISTS 03049_database.r; + +CREATE DATABASE 03049_database; +USE 03049_database; + +CREATE TABLE l (y String) Engine Memory; +CREATE TABLE r (d Date, y String, ty UInt16 MATERIALIZED toYear(d)) Engine Memory; +select * from l L left join r R on L.y = R.y where R.ty >= 2019; +select * from l left join r on l.y = r.y where r.ty >= 2019; +select * from 03049_database.l left join 03049_database.r on l.y = r.y where r.ty >= 2019; + +DROP DATABASE IF EXISTS 03049_database; +DROP TABLE IF EXISTS 03049_database.l; +DROP TABLE IF EXISTS 03049_database.r; From e7e20acc5b13d4a84754d42a38356f3b009531c0 Mon Sep 17 00:00:00 2001 From: justindeguzman Date: Tue, 2 Apr 2024 10:22:57 -0700 Subject: [PATCH 0864/1165] Bump From 51b2a8bce250867a9c54408e2d4f8e72887d68ab Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 17:23:20 +0000 Subject: [PATCH 0865/1165] Close: 111 --- tests/queries/0_stateless/03050_select_one_one_one.reference | 2 ++ tests/queries/0_stateless/03050_select_one_one_one.sql | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 tests/queries/0_stateless/03050_select_one_one_one.reference create mode 100644 tests/queries/0_stateless/03050_select_one_one_one.sql diff --git a/tests/queries/0_stateless/03050_select_one_one_one.reference b/tests/queries/0_stateless/03050_select_one_one_one.reference new file mode 100644 index 00000000000..85e6138dc5d --- /dev/null +++ b/tests/queries/0_stateless/03050_select_one_one_one.reference @@ -0,0 +1,2 @@ +1 1 1 +1 1 1 diff --git a/tests/queries/0_stateless/03050_select_one_one_one.sql b/tests/queries/0_stateless/03050_select_one_one_one.sql new file mode 100644 index 00000000000..eee973fe936 --- /dev/null +++ b/tests/queries/0_stateless/03050_select_one_one_one.sql @@ -0,0 +1,3 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/36973 +SELECT 1, 1, 1; +SELECT * FROM (SELECT 1, 1, 1); From d9048766933a11b283e0e2345d6bc6c9d0a57699 Mon Sep 17 00:00:00 2001 From: justindeguzman Date: Tue, 2 Apr 2024 10:23:50 -0700 Subject: [PATCH 0866/1165] Bump From acf1fb3a6cf3e79899515641848e3214db0c36f6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 17:25:13 +0000 Subject: [PATCH 0867/1165] Forgotten file --- .../03049_unknown_identifier_materialized_column.reference | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/queries/0_stateless/03049_unknown_identifier_materialized_column.reference diff --git a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.reference b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.reference new file mode 100644 index 00000000000..e69de29bb2d From 9ffe6a4640e6197d653c1c5073a98cc659d3f8f6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 17:26:18 +0000 Subject: [PATCH 0868/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/40955 --- tests/queries/0_stateless/03051_many_ctes.reference | 4 ++++ tests/queries/0_stateless/03051_many_ctes.sql | 5 +++++ 2 files changed, 9 insertions(+) create mode 100644 tests/queries/0_stateless/03051_many_ctes.reference create mode 100644 tests/queries/0_stateless/03051_many_ctes.sql diff --git a/tests/queries/0_stateless/03051_many_ctes.reference b/tests/queries/0_stateless/03051_many_ctes.reference new file mode 100644 index 00000000000..487b1165348 --- /dev/null +++ b/tests/queries/0_stateless/03051_many_ctes.reference @@ -0,0 +1,4 @@ +2 +2 +2 +2 diff --git a/tests/queries/0_stateless/03051_many_ctes.sql b/tests/queries/0_stateless/03051_many_ctes.sql new file mode 100644 index 00000000000..412a1e6b544 --- /dev/null +++ b/tests/queries/0_stateless/03051_many_ctes.sql @@ -0,0 +1,5 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/40955 +WITH toInt64(2) AS new_x SELECT new_x AS x FROM (SELECT 1 AS x) t; +WITH toInt64(2) AS new_x SELECT * replace(new_x as x) FROM (SELECT 1 AS x) t; +SELECT 2 AS x FROM (SELECT 1 AS x) t; +SELECT * replace(2 as x) FROM (SELECT 1 AS x) t; From 9673e0a9fcf3006f791da2c0c28cfa4d7b5ac9c7 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 17:29:01 +0000 Subject: [PATCH 0869/1165] Just a test --- ...3052_query_hash_includes_aliases.reference | 2 ++ .../03052_query_hash_includes_aliases.sql | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 tests/queries/0_stateless/03052_query_hash_includes_aliases.reference create mode 100644 tests/queries/0_stateless/03052_query_hash_includes_aliases.sql diff --git a/tests/queries/0_stateless/03052_query_hash_includes_aliases.reference b/tests/queries/0_stateless/03052_query_hash_includes_aliases.reference new file mode 100644 index 00000000000..570d7be9c4b --- /dev/null +++ b/tests/queries/0_stateless/03052_query_hash_includes_aliases.reference @@ -0,0 +1,2 @@ +(1,1) (1,0) +(3,4) (3,11) diff --git a/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql b/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql new file mode 100644 index 00000000000..e76108c7842 --- /dev/null +++ b/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql @@ -0,0 +1,31 @@ +-- https://github.com/ClickHouse/ClickHouse/pull/40065 + +SELECT +( + SELECT + 1 AS number, + number + FROM numbers(1) +) AS s, +( + SELECT + 1, + number + FROM numbers(1) +) AS s2; + +SELECT +( + SELECT + 1 + 2 AS number, + 1 + number AS b + FROM system.numbers + LIMIT 10, 1 +), +( + SELECT + 1 + 2 AS number2, + 1 + number AS b + FROM system.numbers + LIMIT 10, 1 +); From 593abc4037ecc420cd713f62bc166738071ec57b Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 19:36:34 +0200 Subject: [PATCH 0870/1165] Update 03047_analyzer_alias_join.sql --- tests/queries/0_stateless/03047_analyzer_alias_join.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/03047_analyzer_alias_join.sql b/tests/queries/0_stateless/03047_analyzer_alias_join.sql index a6f81516430..ef8c067bb72 100644 --- a/tests/queries/0_stateless/03047_analyzer_alias_join.sql +++ b/tests/queries/0_stateless/03047_analyzer_alias_join.sql @@ -1,4 +1,3 @@ --- https://github.com/ClickHouse/ClickHouse/issues/9954 SELECT 1 AS value, * From 4fc09fb746ef7902d01d54cf9d2f538d168ec9b9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 20:03:56 +0200 Subject: [PATCH 0871/1165] Update 03034_normalized_ast.sql --- tests/queries/0_stateless/03034_normalized_ast.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/03034_normalized_ast.sql b/tests/queries/0_stateless/03034_normalized_ast.sql index ff6f8da118c..fe801cd8cf4 100644 --- a/tests/queries/0_stateless/03034_normalized_ast.sql +++ b/tests/queries/0_stateless/03034_normalized_ast.sql @@ -3,5 +3,6 @@ SELECT concat(database, table) AS name, count() FROM clusterAllReplicas(default, system.tables) +WHERE database=currentDatabase() GROUP BY name FORMAT Null; From e6e49688b25bbd70555a76ae06a89bd1768fcd01 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 2 Apr 2024 18:18:32 +0000 Subject: [PATCH 0872/1165] Fix 02503_insert_storage_snapshot --- tests/queries/0_stateless/02503_insert_storage_snapshot.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/queries/0_stateless/02503_insert_storage_snapshot.sh b/tests/queries/0_stateless/02503_insert_storage_snapshot.sh index b494adeb785..13561947e08 100755 --- a/tests/queries/0_stateless/02503_insert_storage_snapshot.sh +++ b/tests/queries/0_stateless/02503_insert_storage_snapshot.sh @@ -20,6 +20,13 @@ counter=0 retries=60 # for a short period of time. To avoid flakyness we check that refcount became 1 at least once during long INSERT query. # It proves that the INSERT query doesn't hold redundant references to parts. while [[ $counter -lt $retries ]]; do + query_result=$($CLICKHOUSE_CLIENT -q "select count() from system.processes where query_id = '$query_id' FORMAT CSV") + if [ "$query_result" -lt 1 ]; then + sleep 0.1 + ((++counter)) + continue; + fi + query_result=$($CLICKHOUSE_CLIENT -q "SELECT name, active, refcount FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 't_insert_storage_snapshot' FORMAT CSV") if [ "$query_result" == '"all_1_1_0",1,1' ]; then echo "$query_result" From 22fb91a6ef4532104cdee5f1af80c8ab910e147e Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 2 Apr 2024 18:18:32 +0000 Subject: [PATCH 0873/1165] Fix tests --- src/Server/HTTPHandler.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index e754e8b58c3..8ba1e1d6df1 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -902,6 +902,7 @@ void HTTPHandler::processQuery( { bool with_stacktrace = (params.getParsed("stacktrace", false) && server.config().getBool("enable_http_stacktrace", true)); ExecutionStatus status = ExecutionStatus::fromCurrentException("", with_stacktrace); + formatExceptionForClient(status.code, request, response, used_output); current_output_format.setException(status.message); current_output_format.finalize(); used_output.exception_is_written = true; From 69b393cf3eef9d22d180af3e36debf5128b4c27c Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 2 Apr 2024 20:35:50 +0200 Subject: [PATCH 0874/1165] Fxi standalone build --- src/Common/ThreadStatus.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 2d33c0ac021..550cb76e736 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -1,5 +1,6 @@ #pragma once +#include "config.h" #include #include #include @@ -307,7 +308,12 @@ public: void flushUntrackedMemory(); +#ifdef CLICKHOUSE_KEEPER_STANDALONE_BUILD + void initGlobalProfiler(UInt64, UInt64) {} +#else void initGlobalProfiler(UInt64 global_profiler_real_time_period, UInt64 global_profiler_cpu_time_period); +#endif + private: void applyGlobalSettings(); void applyQuerySettings(); From ed448eae426db8c225cf5a6f70f5f780c7820a4b Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 2 Apr 2024 18:40:46 +0000 Subject: [PATCH 0875/1165] add setting lightweight_deletes_sync --- src/Core/Settings.h | 1 + src/Interpreters/InterpreterDeleteQuery.cpp | 2 +- .../03033_lightweight_deletes_sync.reference | 4 ++++ .../03033_lightweight_deletes_sync.sql | 18 ++++++++++++++++++ 4 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03033_lightweight_deletes_sync.reference create mode 100644 tests/queries/0_stateless/03033_lightweight_deletes_sync.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 84e709294aa..be5a38a5bd0 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -590,6 +590,7 @@ class IColumn; M(Bool, optimize_respect_aliases, true, "If it is set to true, it will respect aliases in WHERE/GROUP BY/ORDER BY, that will help with partition pruning/secondary indexes/optimize_aggregation_in_order/optimize_read_in_order/optimize_trivial_count", 0) \ M(UInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \ M(Bool, enable_lightweight_delete, true, "Enable lightweight DELETE mutations for mergetree tables.", 0) ALIAS(allow_experimental_lightweight_delete) \ + M(UInt64, lightweight_deletes_sync, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes", 0) \ M(Bool, apply_deleted_mask, true, "Enables filtering out rows deleted with lightweight DELETE. If disabled, a query will be able to read those rows. This is useful for debugging and \"undelete\" scenarios", 0) \ M(Bool, optimize_normalize_count_variants, true, "Rewrite aggregate functions that semantically equals to count() as count().", 0) \ M(Bool, optimize_injective_functions_inside_uniq, true, "Delete injective functions of one argument inside uniq*() functions.", 0) \ diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 8fb0dabb5b5..07d23be78a7 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -101,7 +101,7 @@ BlockIO InterpreterDeleteQuery::execute() DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); auto context = Context::createCopy(getContext()); - context->setSetting("mutations_sync", 2); /// Lightweight delete is always synchronous + context->setSetting("mutations_sync", Field(context->getSettingsRef().lightweight_deletes_sync)); InterpreterAlterQuery alter_interpreter(alter_ast, context); return alter_interpreter.execute(); } diff --git a/tests/queries/0_stateless/03033_lightweight_deletes_sync.reference b/tests/queries/0_stateless/03033_lightweight_deletes_sync.reference new file mode 100644 index 00000000000..181282ffa9a --- /dev/null +++ b/tests/queries/0_stateless/03033_lightweight_deletes_sync.reference @@ -0,0 +1,4 @@ +2 +0 +2 +1 diff --git a/tests/queries/0_stateless/03033_lightweight_deletes_sync.sql b/tests/queries/0_stateless/03033_lightweight_deletes_sync.sql new file mode 100644 index 00000000000..bb4bb6dfa3d --- /dev/null +++ b/tests/queries/0_stateless/03033_lightweight_deletes_sync.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS t_lightweight_deletes; + +CREATE TABLE t_lightweight_deletes (a UInt64) ENGINE = MergeTree ORDER BY a; + +INSERT INTO t_lightweight_deletes VALUES (1) (2) (3); + +DELETE FROM t_lightweight_deletes WHERE a = 1 SETTINGS lightweight_deletes_sync = 2; + +SELECT count() FROM t_lightweight_deletes; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_lightweight_deletes' AND NOT is_done; + +SYSTEM STOP MERGES t_lightweight_deletes; +DELETE FROM t_lightweight_deletes WHERE a = 2 SETTINGS lightweight_deletes_sync = 0; + +SELECT count() FROM t_lightweight_deletes; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_lightweight_deletes' AND NOT is_done; + +DROP TABLE t_lightweight_deletes; From c2995b13e21b49bf490ef603d237361b639400a9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 2 Apr 2024 20:44:52 +0200 Subject: [PATCH 0876/1165] Use ClickHouse threads in NuRaft --- contrib/NuRaft | 2 +- contrib/nuraft-cmake/CMakeLists.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index 4a12f99dfc9..717657cd94d 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 4a12f99dfc9d47c687ff7700b927cc76856225d1 +Subproject commit 717657cd94da01e86733e58f8d3f0ca0d8748712 diff --git a/contrib/nuraft-cmake/CMakeLists.txt b/contrib/nuraft-cmake/CMakeLists.txt index eaca00566d6..e5d0d6ccb4e 100644 --- a/contrib/nuraft-cmake/CMakeLists.txt +++ b/contrib/nuraft-cmake/CMakeLists.txt @@ -50,6 +50,10 @@ else() target_compile_definitions(_nuraft PRIVATE USE_BOOST_ASIO=1 BOOST_ASIO_STANDALONE=1) endif() +target_link_libraries (_nuraft PRIVATE clickhouse_common_io) +target_compile_definitions(_nuraft PRIVATE USE_CLICKHOUSE_THREADS=1) +MESSAGE(STATUS "Will use clickhouse threads for NuRaft") + target_include_directories (_nuraft SYSTEM PRIVATE "${LIBRARY_DIR}/include/libnuraft") # for some reason include "asio.h" directly without "boost/" prefix. target_include_directories (_nuraft SYSTEM PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") From 9422ea35c5d93f5811361784f9344c2f5e84a6b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 2 Apr 2024 21:24:05 +0200 Subject: [PATCH 0877/1165] Fix non Linux build --- src/Common/AtomicLogger.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/AtomicLogger.h b/src/Common/AtomicLogger.h index 9581358218c..0ece9e8a09a 100644 --- a/src/Common/AtomicLogger.h +++ b/src/Common/AtomicLogger.h @@ -2,9 +2,10 @@ #include +#include #include -#include #include +#include namespace DB { From fb2163ed7c2159b3444a2c6ceec459ee77a08322 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 21:24:13 +0200 Subject: [PATCH 0878/1165] Apply suggestions from code review --- tests/queries/0_stateless/03034_normalized_ast.sql | 2 +- .../queries/0_stateless/03035_alias_column_bug_distributed.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03034_normalized_ast.sql b/tests/queries/0_stateless/03034_normalized_ast.sql index fe801cd8cf4..dd109eb5113 100644 --- a/tests/queries/0_stateless/03034_normalized_ast.sql +++ b/tests/queries/0_stateless/03034_normalized_ast.sql @@ -2,7 +2,7 @@ SELECT concat(database, table) AS name, count() -FROM clusterAllReplicas(default, system.tables) +FROM clusterAllReplicas(test_shard_localhost, system.tables) WHERE database=currentDatabase() GROUP BY name FORMAT Null; diff --git a/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql index fb459b3289b..5fd1f397425 100644 --- a/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql +++ b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql @@ -11,7 +11,7 @@ ORDER BY src; CREATE TABLE alias_bug_dist AS alias_bug -ENGINE = Distributed('default', currentDatabase(), 'alias_bug', rand()); +ENGINE = Distributed('test_shard_localhost', currentDatabase(), 'alias_bug', rand()); INSERT INTO alias_bug VALUES ('SOURCE1'); From 81f210be9328d617e1083283cbd3b654856120fc Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 21:53:03 +0200 Subject: [PATCH 0879/1165] Update tests/queries/0_stateless/03035_alias_column_bug_distributed.sql --- .../queries/0_stateless/03035_alias_column_bug_distributed.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql index 5fd1f397425..3a7b4890bf0 100644 --- a/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql +++ b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql @@ -34,7 +34,7 @@ ORDER BY src; CREATE TABLE alias_bug_dist AS alias_bug -ENGINE = Distributed('default', currentDatabase(), 'alias_bug', rand()); +ENGINE = Distributed('test_shard_localhost', currentDatabase(), 'alias_bug', rand()); -- Unknown identifier SELECT CAST(123, 'String') AS src,theAlias FROM alias_bug_dist LIMIT 1 FORMAT Null; From 606058c1ca489f8fcc77ade96d5d1e39573a0628 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 2 Apr 2024 13:29:07 +0000 Subject: [PATCH 0880/1165] Consolidate SQL compat alias docs into data type docs + improve sidebar order --- .../data-types/aggregatefunction.md | 2 +- docs/en/sql-reference/data-types/array.md | 2 +- docs/en/sql-reference/data-types/boolean.md | 2 +- docs/en/sql-reference/data-types/date.md | 2 +- docs/en/sql-reference/data-types/date32.md | 2 +- docs/en/sql-reference/data-types/datetime.md | 2 +- .../en/sql-reference/data-types/datetime64.md | 2 +- docs/en/sql-reference/data-types/decimal.md | 2 +- docs/en/sql-reference/data-types/enum.md | 2 +- .../sql-reference/data-types/fixedstring.md | 4 +-- docs/en/sql-reference/data-types/float.md | 2 +- docs/en/sql-reference/data-types/geo.md | 4 +-- docs/en/sql-reference/data-types/index.md | 4 +-- docs/en/sql-reference/data-types/int-uint.md | 2 +- docs/en/sql-reference/data-types/ipv4.md | 2 +- docs/en/sql-reference/data-types/ipv6.md | 2 +- docs/en/sql-reference/data-types/json.md | 2 +- .../data-types/lowcardinality.md | 6 ++-- docs/en/sql-reference/data-types/map.md | 8 +++--- .../data-types/multiword-types.md | 27 ------------------ docs/en/sql-reference/data-types/nullable.md | 4 +-- .../data-types/simpleaggregatefunction.md | 2 ++ docs/en/sql-reference/data-types/string.md | 4 +-- docs/en/sql-reference/data-types/tuple.md | 4 +-- docs/en/sql-reference/data-types/uuid.md | 2 +- docs/en/sql-reference/data-types/variant.md | 6 ++-- .../data-types/multiword-types.md | 28 ------------------- .../data-types/multiword-types.mdx | 10 ------- 28 files changed, 39 insertions(+), 102 deletions(-) delete mode 100644 docs/en/sql-reference/data-types/multiword-types.md delete mode 100644 docs/ru/sql-reference/data-types/multiword-types.md delete mode 100644 docs/zh/sql-reference/data-types/multiword-types.mdx diff --git a/docs/en/sql-reference/data-types/aggregatefunction.md b/docs/en/sql-reference/data-types/aggregatefunction.md index fe6d7ebe0dc..87511a505dc 100644 --- a/docs/en/sql-reference/data-types/aggregatefunction.md +++ b/docs/en/sql-reference/data-types/aggregatefunction.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/aggregatefunction -sidebar_position: 53 +sidebar_position: 46 sidebar_label: AggregateFunction --- diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index 0ee7c8de93c..e5a8ce5d18b 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/array -sidebar_position: 52 +sidebar_position: 32 sidebar_label: Array(T) --- diff --git a/docs/en/sql-reference/data-types/boolean.md b/docs/en/sql-reference/data-types/boolean.md index 70abf767a41..4c59bd947de 100644 --- a/docs/en/sql-reference/data-types/boolean.md +++ b/docs/en/sql-reference/data-types/boolean.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/boolean -sidebar_position: 43 +sidebar_position: 22 sidebar_label: Boolean --- diff --git a/docs/en/sql-reference/data-types/date.md b/docs/en/sql-reference/data-types/date.md index 26e4610aec7..7adee3bbf3c 100644 --- a/docs/en/sql-reference/data-types/date.md +++ b/docs/en/sql-reference/data-types/date.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/date -sidebar_position: 47 +sidebar_position: 12 sidebar_label: Date --- diff --git a/docs/en/sql-reference/data-types/date32.md b/docs/en/sql-reference/data-types/date32.md index 38a07cd817d..a08c931b7fc 100644 --- a/docs/en/sql-reference/data-types/date32.md +++ b/docs/en/sql-reference/data-types/date32.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/date32 -sidebar_position: 48 +sidebar_position: 14 sidebar_label: Date32 --- diff --git a/docs/en/sql-reference/data-types/datetime.md b/docs/en/sql-reference/data-types/datetime.md index 1adff18f598..889bc682d91 100644 --- a/docs/en/sql-reference/data-types/datetime.md +++ b/docs/en/sql-reference/data-types/datetime.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/datetime -sidebar_position: 48 +sidebar_position: 16 sidebar_label: DateTime --- diff --git a/docs/en/sql-reference/data-types/datetime64.md b/docs/en/sql-reference/data-types/datetime64.md index 504d0e2b0a6..ef452a723e6 100644 --- a/docs/en/sql-reference/data-types/datetime64.md +++ b/docs/en/sql-reference/data-types/datetime64.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/datetime64 -sidebar_position: 49 +sidebar_position: 18 sidebar_label: DateTime64 --- diff --git a/docs/en/sql-reference/data-types/decimal.md b/docs/en/sql-reference/data-types/decimal.md index 2b32e72a28f..dfdefdff5a5 100644 --- a/docs/en/sql-reference/data-types/decimal.md +++ b/docs/en/sql-reference/data-types/decimal.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/decimal -sidebar_position: 42 +sidebar_position: 6 sidebar_label: Decimal --- diff --git a/docs/en/sql-reference/data-types/enum.md b/docs/en/sql-reference/data-types/enum.md index 02e73a0360e..ccfeb7f3416 100644 --- a/docs/en/sql-reference/data-types/enum.md +++ b/docs/en/sql-reference/data-types/enum.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/enum -sidebar_position: 50 +sidebar_position: 20 sidebar_label: Enum --- diff --git a/docs/en/sql-reference/data-types/fixedstring.md b/docs/en/sql-reference/data-types/fixedstring.md index a56b3fccbc1..0316df7fe34 100644 --- a/docs/en/sql-reference/data-types/fixedstring.md +++ b/docs/en/sql-reference/data-types/fixedstring.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/fixedstring -sidebar_position: 45 +sidebar_position: 10 sidebar_label: FixedString(N) --- -# FixedString +# FixedString(N) A fixed-length string of `N` bytes (neither characters nor code points). diff --git a/docs/en/sql-reference/data-types/float.md b/docs/en/sql-reference/data-types/float.md index be7b2a7fcd8..23131d5b4fe 100644 --- a/docs/en/sql-reference/data-types/float.md +++ b/docs/en/sql-reference/data-types/float.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/float -sidebar_position: 41 +sidebar_position: 4 sidebar_label: Float32, Float64 --- diff --git a/docs/en/sql-reference/data-types/geo.md b/docs/en/sql-reference/data-types/geo.md index 1d37b829dd5..7e3c32b3451 100644 --- a/docs/en/sql-reference/data-types/geo.md +++ b/docs/en/sql-reference/data-types/geo.md @@ -1,8 +1,8 @@ --- slug: /en/sql-reference/data-types/geo -sidebar_position: 62 +sidebar_position: 54 sidebar_label: Geo -title: "Geo Data Types" +title: "Geometric" --- ClickHouse supports data types for representing geographical objects — locations, lands, etc. diff --git a/docs/en/sql-reference/data-types/index.md b/docs/en/sql-reference/data-types/index.md index ffd063590fa..fcb0b60d022 100644 --- a/docs/en/sql-reference/data-types/index.md +++ b/docs/en/sql-reference/data-types/index.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/ sidebar_label: List of data types -sidebar_position: 37 +sidebar_position: 1 --- -# ClickHouse Data Types +# Data Types in ClickHouse ClickHouse can store various kinds of data in table cells. This section describes the supported data types and special considerations for using and/or implementing them if any. diff --git a/docs/en/sql-reference/data-types/int-uint.md b/docs/en/sql-reference/data-types/int-uint.md index 520454a859f..52d2982de19 100644 --- a/docs/en/sql-reference/data-types/int-uint.md +++ b/docs/en/sql-reference/data-types/int-uint.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/int-uint -sidebar_position: 40 +sidebar_position: 2 sidebar_label: UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 --- diff --git a/docs/en/sql-reference/data-types/ipv4.md b/docs/en/sql-reference/data-types/ipv4.md index 288806f47b3..637ed543e08 100644 --- a/docs/en/sql-reference/data-types/ipv4.md +++ b/docs/en/sql-reference/data-types/ipv4.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/ipv4 -sidebar_position: 59 +sidebar_position: 28 sidebar_label: IPv4 --- diff --git a/docs/en/sql-reference/data-types/ipv6.md b/docs/en/sql-reference/data-types/ipv6.md index 97959308b58..642a7db81fc 100644 --- a/docs/en/sql-reference/data-types/ipv6.md +++ b/docs/en/sql-reference/data-types/ipv6.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/ipv6 -sidebar_position: 60 +sidebar_position: 30 sidebar_label: IPv6 --- diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md index fd548a0d5a2..39e37abad82 100644 --- a/docs/en/sql-reference/data-types/json.md +++ b/docs/en/sql-reference/data-types/json.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/json -sidebar_position: 54 +sidebar_position: 26 sidebar_label: JSON --- diff --git a/docs/en/sql-reference/data-types/lowcardinality.md b/docs/en/sql-reference/data-types/lowcardinality.md index db10103282d..133ac2bd72e 100644 --- a/docs/en/sql-reference/data-types/lowcardinality.md +++ b/docs/en/sql-reference/data-types/lowcardinality.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/lowcardinality -sidebar_position: 51 -sidebar_label: LowCardinality +sidebar_position: 42 +sidebar_label: LowCardinality(T) --- -# LowCardinality +# LowCardinality(T) Changes the internal representation of other data types to be dictionary-encoded. diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md index e0c8b98f9f8..2c734969afc 100644 --- a/docs/en/sql-reference/data-types/map.md +++ b/docs/en/sql-reference/data-types/map.md @@ -1,12 +1,12 @@ --- slug: /en/sql-reference/data-types/map -sidebar_position: 65 -sidebar_label: Map(key, value) +sidebar_position: 36 +sidebar_label: Map(K, V) --- -# Map(key, value) +# Map(K, V) -`Map(key, value)` data type stores `key:value` pairs. +`Map(K, V)` data type stores `key:value` pairs. **Parameters** diff --git a/docs/en/sql-reference/data-types/multiword-types.md b/docs/en/sql-reference/data-types/multiword-types.md deleted file mode 100644 index ebbe1d84544..00000000000 --- a/docs/en/sql-reference/data-types/multiword-types.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -slug: /en/sql-reference/data-types/multiword-types -sidebar_position: 61 -sidebar_label: Multiword Type Names -title: "Multiword Types" ---- - -When creating tables, you can use data types with a name consisting of several words. This is implemented for better SQL compatibility. - -## Multiword Types Support - -| Multiword types | Simple types | -|----------------------------------|--------------------------------------------------------------| -| DOUBLE PRECISION | [Float64](../../sql-reference/data-types/float.md) | -| CHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| CHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | -| NCHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| NCHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHAR | [String](../../sql-reference/data-types/string.md) | -| BINARY LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| BINARY VARYING | [String](../../sql-reference/data-types/string.md) | diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 5504765e4a0..abcb87a0c1b 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -1,7 +1,7 @@ --- slug: /en/sql-reference/data-types/nullable -sidebar_position: 55 -sidebar_label: Nullable +sidebar_position: 44 +sidebar_label: Nullable(T) --- # Nullable(T) diff --git a/docs/en/sql-reference/data-types/simpleaggregatefunction.md b/docs/en/sql-reference/data-types/simpleaggregatefunction.md index 517a28576f0..39f8409c1e1 100644 --- a/docs/en/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/en/sql-reference/data-types/simpleaggregatefunction.md @@ -1,5 +1,7 @@ --- slug: /en/sql-reference/data-types/simpleaggregatefunction +sidebar_position: 48 +sidebar_label: SimpleAggregateFunction --- # SimpleAggregateFunction diff --git a/docs/en/sql-reference/data-types/string.md b/docs/en/sql-reference/data-types/string.md index f891a9303e5..8a4f346fdfc 100644 --- a/docs/en/sql-reference/data-types/string.md +++ b/docs/en/sql-reference/data-types/string.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/string -sidebar_position: 44 +sidebar_position: 8 sidebar_label: String --- @@ -13,7 +13,7 @@ When creating tables, numeric parameters for string fields can be set (e.g. `VAR Aliases: -- `String` — `LONGTEXT`, `MEDIUMTEXT`, `TINYTEXT`, `TEXT`, `LONGBLOB`, `MEDIUMBLOB`, `TINYBLOB`, `BLOB`, `VARCHAR`, `CHAR`. +- `String` — `LONGTEXT`, `MEDIUMTEXT`, `TINYTEXT`, `TEXT`, `LONGBLOB`, `MEDIUMBLOB`, `TINYBLOB`, `BLOB`, `VARCHAR`, `CHAR`, `CHAR LARGE OBJECT`, `CHAR VARYING`, `CHARACTER LARGE OBJECT`, `CHARACTER VARYING`, `NCHAR LARGE OBJECT`, `NCHAR VARYING`, `NATIONAL CHARACTER LARGE OBJECT`, `NATIONAL CHARACTER VARYING`, `NATIONAL CHAR VARYING`, `NATIONAL CHARACTER`, `NATIONAL CHAR`, `BINARY LARGE OBJECT`, `BINARY VARYING`, ## Encodings diff --git a/docs/en/sql-reference/data-types/tuple.md b/docs/en/sql-reference/data-types/tuple.md index 8f87eeca075..0525a3b0476 100644 --- a/docs/en/sql-reference/data-types/tuple.md +++ b/docs/en/sql-reference/data-types/tuple.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/tuple -sidebar_position: 54 +sidebar_position: 34 sidebar_label: Tuple(T1, T2, ...) --- -# Tuple(T1, T2, …) +# Tuple(T1, T2, ...) A tuple of elements, each having an individual [type](../../sql-reference/data-types/index.md#data_types). Tuple must contain at least one element. diff --git a/docs/en/sql-reference/data-types/uuid.md b/docs/en/sql-reference/data-types/uuid.md index 40f756b9588..75e163f5063 100644 --- a/docs/en/sql-reference/data-types/uuid.md +++ b/docs/en/sql-reference/data-types/uuid.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/uuid -sidebar_position: 46 +sidebar_position: 24 sidebar_label: UUID --- diff --git a/docs/en/sql-reference/data-types/variant.md b/docs/en/sql-reference/data-types/variant.md index 7d10d4b0e97..1a9f1dde8d3 100644 --- a/docs/en/sql-reference/data-types/variant.md +++ b/docs/en/sql-reference/data-types/variant.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/variant -sidebar_position: 55 -sidebar_label: Variant +sidebar_position: 40 +sidebar_label: Variant(T1, T2, ...) --- -# Variant(T1, T2, T3, ...) +# Variant(T1, T2, ...) This type represents a union of other data types. Type `Variant(T1, T2, ..., TN)` means that each row of this type has a value of either type `T1` or `T2` or ... or `TN` or none of them (`NULL` value). diff --git a/docs/ru/sql-reference/data-types/multiword-types.md b/docs/ru/sql-reference/data-types/multiword-types.md deleted file mode 100644 index cca2d71e480..00000000000 --- a/docs/ru/sql-reference/data-types/multiword-types.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -slug: /ru/sql-reference/data-types/multiword-types -sidebar_position: 61 -sidebar_label: Составные типы ---- - -# Составные типы {#multiword-types} - -При создании таблиц вы можете использовать типы данных с названием, состоящим из нескольких слов. Такие названия поддерживаются для лучшей совместимости с SQL. - -## Поддержка составных типов {#multiword-types-support} - -| Составные типы | Обычные типы | -|-------------------------------------|-----------------------------------------------------------| -| DOUBLE PRECISION | [Float64](../../sql-reference/data-types/float.md) | -| CHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| CHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | -| NCHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| NCHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHAR | [String](../../sql-reference/data-types/string.md) | -| BINARY LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| BINARY VARYING | [String](../../sql-reference/data-types/string.md) | diff --git a/docs/zh/sql-reference/data-types/multiword-types.mdx b/docs/zh/sql-reference/data-types/multiword-types.mdx deleted file mode 100644 index 85431d47efd..00000000000 --- a/docs/zh/sql-reference/data-types/multiword-types.mdx +++ /dev/null @@ -1,10 +0,0 @@ ---- -slug: /zh/sql-reference/data-types/multiword-types -sidebar_position: 61 -sidebar_label: Multiword Type Names -title: "Multiword Types" ---- - -import Content from '@site/docs/en/sql-reference/data-types/multiword-types.md'; - - From cf8a83bb36a5b4072b100301abf267d74d8a4874 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Apr 2024 21:05:37 +0000 Subject: [PATCH 0881/1165] Bump From 0ad3ed5ae3deb4e5077a8d18cf9fb16be1495d01 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 2 Apr 2024 18:38:58 -0300 Subject: [PATCH 0882/1165] Add support for S3 access through aws private link interface --- src/IO/S3/URI.cpp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 027cb624ed5..b007d9268b7 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -33,12 +33,18 @@ namespace S3 URI::URI(const std::string & uri_) { /// Case when bucket name represented in domain name of S3 URL. - /// E.g. (https://bucket-name.s3.Region.amazonaws.com/key) + /// E.g. (https://bucket-name.s3.region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3express[\-a-z0-9]+|s3|cos|obs|oss|eos)([.\-][a-z0-9\-.:]+))"); + /// Case when AWS Private Link Interface is being used + /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com) + /// https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html +// static const RE2 aws_private_link_style_pattern("bucket\\.vpce\\-([a-z0-9\\-.:]+)\\.vpce.amazonaws.com/([^/]*)/(.*)"); + static const RE2 aws_private_link_style_pattern("bucket\\.vpce\\-([a-z0-9\\-.:]+)\\.vpce.amazonaws.com"); + /// Case when bucket name and key represented in path of S3 URL. - /// E.g. (https://s3.Region.amazonaws.com/bucket-name/key) + /// E.g. (https://s3.region.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#path-style-access static const RE2 path_style_pattern("^/([^/]*)/(.*)"); @@ -103,7 +109,19 @@ URI::URI(const std::string & uri_) String name; String endpoint_authority_from_uri; - if (re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &name, &endpoint_authority_from_uri)) + if (re2::RE2::FullMatch(uri.getAuthority(), aws_private_link_style_pattern)) + { + if (!re2::RE2::PartialMatch(uri.getPath(), path_style_pattern, &bucket, &key)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Object storage system name is unrecognized in virtual hosted style S3 URI: {}", + quoteString("ada")); + } + is_virtual_hosted_style = true; + endpoint = uri.getScheme() + "://" + uri.getAuthority(); + validateBucket(bucket, uri); + } + else if (re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &name, &endpoint_authority_from_uri)) { is_virtual_hosted_style = true; endpoint = uri.getScheme() + "://" + name + endpoint_authority_from_uri; From c1fd9262a7126fa9a8cbd3e18942eb499ce93965 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 2 Apr 2024 18:42:57 -0300 Subject: [PATCH 0883/1165] change exception placeholder --- src/IO/S3/URI.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index b007d9268b7..69b539cde8b 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -38,9 +38,8 @@ URI::URI(const std::string & uri_) static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3express[\-a-z0-9]+|s3|cos|obs|oss|eos)([.\-][a-z0-9\-.:]+))"); /// Case when AWS Private Link Interface is being used - /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com) + /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html -// static const RE2 aws_private_link_style_pattern("bucket\\.vpce\\-([a-z0-9\\-.:]+)\\.vpce.amazonaws.com/([^/]*)/(.*)"); static const RE2 aws_private_link_style_pattern("bucket\\.vpce\\-([a-z0-9\\-.:]+)\\.vpce.amazonaws.com"); /// Case when bucket name and key represented in path of S3 URL. @@ -113,10 +112,10 @@ URI::URI(const std::string & uri_) { if (!re2::RE2::PartialMatch(uri.getPath(), path_style_pattern, &bucket, &key)) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Object storage system name is unrecognized in virtual hosted style S3 URI: {}", - quoteString("ada")); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Could not parse bucket and key from uri {}", uri.toString()); } + + // Default to virtual hosted style is_virtual_hosted_style = true; endpoint = uri.getScheme() + "://" + uri.getAuthority(); validateBucket(bucket, uri); From 44fa62c0a49c1dd847a60b4863c6cfe949e40672 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 30 Mar 2024 01:28:14 +0100 Subject: [PATCH 0884/1165] Add test. --- src/Common/randomDelay.cpp | 39 ++++++ src/Common/randomDelay.h | 8 ++ src/Storages/MergeTree/DataPartsExchange.cpp | 5 + src/Storages/StorageReplicatedMergeTree.cpp | 5 + .../{remote_servers.xml => cluster.xml} | 16 --- .../configs/cluster3.xml | 20 +++ .../configs/slow_replicated_merge_tree.xml | 10 ++ .../test_backup_restore_on_cluster/test.py | 3 +- .../test_slow_rmt.py | 119 ++++++++++++++++++ 9 files changed, 208 insertions(+), 17 deletions(-) create mode 100644 src/Common/randomDelay.cpp create mode 100644 src/Common/randomDelay.h rename tests/integration/test_backup_restore_on_cluster/configs/{remote_servers.xml => cluster.xml} (56%) create mode 100644 tests/integration/test_backup_restore_on_cluster/configs/cluster3.xml create mode 100644 tests/integration/test_backup_restore_on_cluster/configs/slow_replicated_merge_tree.xml create mode 100644 tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py diff --git a/src/Common/randomDelay.cpp b/src/Common/randomDelay.cpp new file mode 100644 index 00000000000..7f6f3084919 --- /dev/null +++ b/src/Common/randomDelay.cpp @@ -0,0 +1,39 @@ +#include + +#include +#include +#include + + +void randomDelayForMaxMilliseconds(uint64_t milliseconds, LoggerPtr log, const char * start_of_message) +{ + if (milliseconds) + { + auto count = randomNumber() % milliseconds; + + if (log) + { + if (start_of_message && !*start_of_message) + start_of_message = nullptr; + + LOG_TEST(log, "{}{}Sleeping for {} milliseconds", + (start_of_message ? start_of_message : ""), + (start_of_message ? ": " : ""), + count); + } + + sleepForMilliseconds(count); + + if (log) + { + LOG_TEST(log, "{}{}Awaking after sleeping", + (start_of_message ? start_of_message : ""), + (start_of_message ? ": " : "")); + } + } +} + +void randomDelayForMaxSeconds(uint64_t seconds, LoggerPtr log, const char * start_of_message) +{ + randomDelayForMaxMilliseconds(seconds * 1000, log, start_of_message); +} diff --git a/src/Common/randomDelay.h b/src/Common/randomDelay.h new file mode 100644 index 00000000000..99f218cc8a1 --- /dev/null +++ b/src/Common/randomDelay.h @@ -0,0 +1,8 @@ +#pragma once + +#include + +/// Sleeps for random duration between 0 and a specified number of milliseconds, optionally outputs a logging message about that. +/// This function can be used to add random delays in tests. +void randomDelayForMaxMilliseconds(uint64_t milliseconds, LoggerPtr log = nullptr, const char * start_of_message = nullptr); +void randomDelayForMaxSeconds(uint64_t seconds, LoggerPtr log = nullptr, const char * start_of_message = nullptr); diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 6bb5ff5a4ab..91444d76a52 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -120,6 +121,10 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write LOG_TRACE(log, "Sending part {}", part_name); + static const auto test_delay = data.getContext()->getConfigRef().getUInt64("test.data_parts_exchange.delay_before_sending_part_ms", 0); + if (test_delay) + randomDelayForMaxMilliseconds(test_delay, log, "DataPartsExchange: Before sending part"); + MergeTreeData::DataPartPtr part; auto report_broken_part = [&]() diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 7ca508c362d..4de53d90a33 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -3093,6 +3094,10 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo Strings active_parts = get_part_set.getParts(); /// Remove local parts if source replica does not have them, because such parts will never be fetched by other replicas. + static const auto test_delay = getContext()->getConfigRef().getUInt64("test.clone_replica.delay_before_removing_local_parts_ms", 0); + if (test_delay) + randomDelayForMaxMilliseconds(test_delay, log.load(), "cloneReplica: Before removing local parts"); + Strings local_parts_in_zk = zookeeper->getChildren(fs::path(replica_path) / "parts"); Strings parts_to_remove_from_zk; diff --git a/tests/integration/test_backup_restore_on_cluster/configs/remote_servers.xml b/tests/integration/test_backup_restore_on_cluster/configs/cluster.xml similarity index 56% rename from tests/integration/test_backup_restore_on_cluster/configs/remote_servers.xml rename to tests/integration/test_backup_restore_on_cluster/configs/cluster.xml index c3bb226c1f4..1f7cb8155eb 100644 --- a/tests/integration/test_backup_restore_on_cluster/configs/remote_servers.xml +++ b/tests/integration/test_backup_restore_on_cluster/configs/cluster.xml @@ -20,21 +20,5 @@ - - - - node1 - 9000 - - - node2 - 9000 - - - node3 - 9000 - - - diff --git a/tests/integration/test_backup_restore_on_cluster/configs/cluster3.xml b/tests/integration/test_backup_restore_on_cluster/configs/cluster3.xml new file mode 100644 index 00000000000..a591f22447e --- /dev/null +++ b/tests/integration/test_backup_restore_on_cluster/configs/cluster3.xml @@ -0,0 +1,20 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + node3 + 9000 + + + + + diff --git a/tests/integration/test_backup_restore_on_cluster/configs/slow_replicated_merge_tree.xml b/tests/integration/test_backup_restore_on_cluster/configs/slow_replicated_merge_tree.xml new file mode 100644 index 00000000000..c6bc1e31802 --- /dev/null +++ b/tests/integration/test_backup_restore_on_cluster/configs/slow_replicated_merge_tree.xml @@ -0,0 +1,10 @@ + + + + 250 + + + 250 + + + diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index c76e9718640..e8d24934c7f 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -10,7 +10,8 @@ from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) main_configs = [ - "configs/remote_servers.xml", + "configs/cluster.xml", + "configs/cluster3.xml", "configs/replicated_access_storage.xml", "configs/replicated_user_defined_sql_objects.xml", "configs/backups_disk.xml", diff --git a/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py b/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py new file mode 100644 index 00000000000..987f8669488 --- /dev/null +++ b/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py @@ -0,0 +1,119 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry, exec_query_with_retry + + +cluster = ClickHouseCluster(__file__) + +main_configs = [ + "configs/backups_disk.xml", + "configs/cluster.xml", + "configs/slow_replicated_merge_tree.xml", +] + +user_configs = [ + "configs/allow_database_types.xml", + "configs/zookeeper_retries.xml", +] + +node1 = cluster.add_instance( + "node1", + main_configs=main_configs, + user_configs=user_configs, + external_dirs=["/backups/"], + macros={"replica": "node1", "shard": "shard1"}, + with_zookeeper=True, +) + +node2 = cluster.add_instance( + "node2", + main_configs=main_configs, + user_configs=user_configs, + external_dirs=["/backups/"], + macros={"replica": "node2", "shard": "shard1"}, + with_zookeeper=True, +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def drop_after_test(): + try: + yield + finally: + node1.query("DROP DATABASE IF EXISTS mydb ON CLUSTER 'cluster' SYNC") + + +backup_id_counter = 0 + + +def new_backup_name(): + global backup_id_counter + backup_id_counter += 1 + return f"Disk('backups', '{backup_id_counter}')" + + +def test_replicated_database_async(): + node1.query( + "CREATE DATABASE mydb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}')" + ) + + node1.query("CREATE TABLE mydb.tbl(x UInt8) ENGINE=ReplicatedMergeTree ORDER BY x") + + node1.query( + "CREATE TABLE mydb.tbl2(y String) ENGINE=ReplicatedMergeTree ORDER BY y" + ) + + node2.query("SYSTEM SYNC DATABASE REPLICA mydb") + + node1.query("INSERT INTO mydb.tbl VALUES (1)") + node1.query("INSERT INTO mydb.tbl VALUES (22)") + node2.query("INSERT INTO mydb.tbl2 VALUES ('a')") + node2.query("INSERT INTO mydb.tbl2 VALUES ('bb')") + node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' mydb.tbl") + + backup_name = new_backup_name() + [id, status] = node1.query( + f"BACKUP DATABASE mydb ON CLUSTER 'cluster' TO {backup_name} ASYNC" + ).split("\t") + + assert status == "CREATING_BACKUP\n" or status == "BACKUP_CREATED\n" + + assert_eq_with_retry( + node1, + f"SELECT status, error FROM system.backups WHERE id='{id}'", + TSV([["BACKUP_CREATED", ""]]), + ) + + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' SYNC") + + [id, status] = node1.query( + f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} ASYNC" + ).split("\t") + + assert status == "RESTORING\n" or status == "RESTORED\n" + + assert_eq_with_retry( + node1, + f"SELECT status, error FROM system.backups WHERE id='{id}'", + TSV([["RESTORED", ""]]), + ) + + # exec_query_with_retry() is here because `SYSTEM SYNC REPLICA` can throw `TABLE_IS_READ_ONLY` + # if any of these tables didn't start completely yet. + exec_query_with_retry(node1, "SYSTEM SYNC REPLICA ON CLUSTER 'cluster' mydb.tbl") + exec_query_with_retry(node1, "SYSTEM SYNC REPLICA ON CLUSTER 'cluster' mydb.tbl2") + + assert node1.query("SELECT * FROM mydb.tbl ORDER BY x") == TSV([1, 22]) + assert node2.query("SELECT * FROM mydb.tbl2 ORDER BY y") == TSV(["a", "bb"]) + assert node2.query("SELECT * FROM mydb.tbl ORDER BY x") == TSV([1, 22]) + assert node1.query("SELECT * FROM mydb.tbl2 ORDER BY y") == TSV(["a", "bb"]) From 534905ff8c2cd90cda863a07d4d8510365e99f32 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 2 Apr 2024 22:14:58 +0000 Subject: [PATCH 0885/1165] fix test --- src/Core/SettingsChangesHistory.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 768b6aa6cbd..f43ca154d56 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -86,6 +86,7 @@ namespace SettingsChangesHistory static std::map settings_changes_history = { {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, + {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"}, }}, {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, From a7c490e1dfeadc6f968a57aca147683663fbe766 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Apr 2024 05:42:41 +0200 Subject: [PATCH 0886/1165] Fix build with clang-19 (master) --- base/poco/JSON/src/pdjson.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/poco/JSON/src/pdjson.c b/base/poco/JSON/src/pdjson.c index 18768ac96d3..563fa277439 100644 --- a/base/poco/JSON/src/pdjson.c +++ b/base/poco/JSON/src/pdjson.c @@ -314,13 +314,13 @@ static int read_unicode(json_stream *json) if (l < 0xdc00 || l > 0xdfff) { json_error(json, "invalid surrogate pair continuation \\u%04lx out " - "of range (dc00-dfff)", l); + "of range (dc00-dfff)", (unsigned long)l); return -1; } cp = ((h - 0xd800) * 0x400) + ((l - 0xdc00) + 0x10000); } else if (cp >= 0xdc00 && cp <= 0xdfff) { - json_error(json, "dangling surrogate \\u%04lx", cp); + json_error(json, "dangling surrogate \\u%04lx", (unsigned long)cp); return -1; } From cc31b837f2d9fc44334d831a24898e1321b50134 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 2 Apr 2024 15:28:57 +0200 Subject: [PATCH 0887/1165] User specific S3 endpoint backup/restore ON CLUSTER --- src/Backups/BackupIO_S3.cpp | 10 +++-- src/Backups/BackupIO_S3.h | 21 +++++++++- src/Backups/BackupsWorker.cpp | 1 + src/Backups/registerBackupEngineS3.cpp | 6 ++- src/Storages/StorageS3Settings.cpp | 4 +- src/Storages/StorageS3Settings.h | 2 +- .../configs/remote_servers.xml | 12 ++++++ .../test_backup_restore_s3/test.py | 40 ++++++++++++++++++- 8 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 tests/integration/test_backup_restore_s3/configs/remote_servers.xml diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 2063af2061c..4b7e3d1e775 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -124,11 +124,12 @@ BackupReaderS3::BackupReaderS3( bool allow_s3_native_copy, const ReadSettings & read_settings_, const WriteSettings & write_settings_, - const ContextPtr & context_) + const ContextPtr & context_, + bool is_internal_backup) : BackupReaderDefault(read_settings_, write_settings_, getLogger("BackupReaderS3")) , s3_uri(s3_uri_) , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::S3, MetadataStorageType::None, s3_uri.endpoint, false, false} - , s3_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString(), context_->getUserName())) + , s3_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString(), context_->getUserName(), /*ignore_user=*/is_internal_backup)) { auto & request_settings = s3_settings.request_settings; request_settings.updateFromSettings(context_->getSettingsRef()); @@ -214,11 +215,12 @@ BackupWriterS3::BackupWriterS3( const String & storage_class_name, const ReadSettings & read_settings_, const WriteSettings & write_settings_, - const ContextPtr & context_) + const ContextPtr & context_, + bool is_internal_backup) : BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterS3")) , s3_uri(s3_uri_) , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::S3, MetadataStorageType::None, s3_uri.endpoint, false, false} - , s3_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString(), context_->getUserName())) + , s3_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString(), context_->getUserName(), /*ignore_user=*/is_internal_backup)) { auto & request_settings = s3_settings.request_settings; request_settings.updateFromSettings(context_->getSettingsRef()); diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h index 57108d122ea..f81eb975df3 100644 --- a/src/Backups/BackupIO_S3.h +++ b/src/Backups/BackupIO_S3.h @@ -18,7 +18,15 @@ namespace DB class BackupReaderS3 : public BackupReaderDefault { public: - BackupReaderS3(const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, bool allow_s3_native_copy, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_); + BackupReaderS3( + const S3::URI & s3_uri_, + const String & access_key_id_, + const String & secret_access_key_, + bool allow_s3_native_copy, + const ReadSettings & read_settings_, + const WriteSettings & write_settings_, + const ContextPtr & context_, + bool is_internal_backup); ~BackupReaderS3() override; bool fileExists(const String & file_name) override; @@ -41,7 +49,16 @@ private: class BackupWriterS3 : public BackupWriterDefault { public: - BackupWriterS3(const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, bool allow_s3_native_copy, const String & storage_class_name, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_); + BackupWriterS3( + const S3::URI & s3_uri_, + const String & access_key_id_, + const String & secret_access_key_, + bool allow_s3_native_copy, + const String & storage_class_name, + const ReadSettings & read_settings_, + const WriteSettings & write_settings_, + const ContextPtr & context_, + bool is_internal_backup); ~BackupWriterS3() override; bool fileExists(const String & file_name) override; diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 96fe770227c..d345223b3b4 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -940,6 +940,7 @@ void BackupsWorker::doRestore( backup_open_params.use_same_s3_credentials_for_base_backup = restore_settings.use_same_s3_credentials_for_base_backup; backup_open_params.read_settings = getReadSettingsForRestore(context); backup_open_params.write_settings = getWriteSettingsForRestore(context); + backup_open_params.is_internal_backup = restore_settings.internal; BackupPtr backup = BackupFactory::instance().createBackup(backup_open_params); String current_database = context->getCurrentDatabase(); diff --git a/src/Backups/registerBackupEngineS3.cpp b/src/Backups/registerBackupEngineS3.cpp index fed5c6b4d22..c34dbe273f5 100644 --- a/src/Backups/registerBackupEngineS3.cpp +++ b/src/Backups/registerBackupEngineS3.cpp @@ -110,7 +110,8 @@ void registerBackupEngineS3(BackupFactory & factory) params.allow_s3_native_copy, params.read_settings, params.write_settings, - params.context); + params.context, + params.is_internal_backup); return std::make_unique( params.backup_info, @@ -129,7 +130,8 @@ void registerBackupEngineS3(BackupFactory & factory) params.s3_storage_class, params.read_settings, params.write_settings, - params.context); + params.context, + params.is_internal_backup); return std::make_unique( params.backup_info, diff --git a/src/Storages/StorageS3Settings.cpp b/src/Storages/StorageS3Settings.cpp index 5887018268b..04634bcf1b3 100644 --- a/src/Storages/StorageS3Settings.cpp +++ b/src/Storages/StorageS3Settings.cpp @@ -292,7 +292,7 @@ void StorageS3Settings::loadFromConfig(const String & config_elem, const Poco::U } } -S3Settings StorageS3Settings::getSettings(const String & endpoint, const String & user) const +S3Settings StorageS3Settings::getSettings(const String & endpoint, const String & user, bool ignore_user) const { std::lock_guard lock(mutex); auto next_prefix_setting = s3_settings.upper_bound(endpoint); @@ -302,7 +302,7 @@ S3Settings StorageS3Settings::getSettings(const String & endpoint, const String { std::advance(possible_prefix_setting, -1); const auto & [endpoint_prefix, settings] = *possible_prefix_setting; - if (endpoint.starts_with(endpoint_prefix) && settings.auth_settings.canBeUsedByUser(user)) + if (endpoint.starts_with(endpoint_prefix) && (ignore_user || settings.auth_settings.canBeUsedByUser(user))) return possible_prefix_setting->second; } diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index 21b6264717e..0f972db02b1 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -112,7 +112,7 @@ class StorageS3Settings public: void loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config, const Settings & settings); - S3Settings getSettings(const String & endpoint, const String & user) const; + S3Settings getSettings(const String & endpoint, const String & user, bool ignore_user = false) const; private: mutable std::mutex mutex; diff --git a/tests/integration/test_backup_restore_s3/configs/remote_servers.xml b/tests/integration/test_backup_restore_s3/configs/remote_servers.xml new file mode 100644 index 00000000000..9607aac2003 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/configs/remote_servers.xml @@ -0,0 +1,12 @@ + + + + + + node + 9000 + + + + + diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index d65fc1f09d6..05424887736 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -1,4 +1,4 @@ -from typing import Dict, Iterable +from typing import Dict import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV @@ -13,11 +13,13 @@ node = cluster.add_instance( "configs/named_collection_s3_backups.xml", "configs/s3_settings.xml", "configs/blob_log.xml", + "configs/remote_servers.xml", ], user_configs=[ "configs/zookeeper_retries.xml", ], with_minio=True, + with_zookeeper=True, ) @@ -544,9 +546,45 @@ def test_user_specific_auth(start_cluster): "SELECT * FROM s3('http://minio1:9001/root/data/backups/limited/backup1.zip', 'RawBLOB')", user="regularuser", ) + node.query( "SELECT * FROM s3('http://minio1:9001/root/data/backups/limited/backup1.zip', 'RawBLOB')", user="superuser1", ) + assert "Access Denied" in node.query_and_get_error( + "BACKUP TABLE specific_auth ON CLUSTER 'cluster' TO S3('http://minio1:9001/root/data/backups/limited/backup3/')", + user="regularuser", + ) + + node.query( + "BACKUP TABLE specific_auth ON CLUSTER 'cluster' TO S3('http://minio1:9001/root/data/backups/limited/backup3/')", + user="superuser1", + ) + + assert "Access Denied" in node.query_and_get_error( + "RESTORE TABLE specific_auth ON CLUSTER 'cluster' FROM S3('http://minio1:9001/root/data/backups/limited/backup3/')", + user="regularuser", + ) + + node.query( + "RESTORE TABLE specific_auth ON CLUSTER 'cluster' FROM S3('http://minio1:9001/root/data/backups/limited/backup3/')", + user="superuser1", + ) + + assert "Access Denied" in node.query_and_get_error( + "SELECT * FROM s3('http://minio1:9001/root/data/backups/limited/backup3/*', 'RawBLOB')", + user="regularuser", + ) + + node.query( + "SELECT * FROM s3('http://minio1:9001/root/data/backups/limited/backup3/*', 'RawBLOB')", + user="superuser1", + ) + + assert "Access Denied" in node.query_and_get_error( + "SELECT * FROM s3Cluster(cluster, 'http://minio1:9001/root/data/backups/limited/backup3/*', 'RawBLOB')", + user="regularuser", + ) + node.query("DROP TABLE IF EXISTS test.specific_auth") From a6356c120ab39db6af18e175856cc11b4d8fed00 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Apr 2024 09:22:01 +0200 Subject: [PATCH 0888/1165] Add more documentation to the release script --- tests/ci/release.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/ci/release.py b/tests/ci/release.py index 679e65560f5..b7ccc59f7c1 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -7,6 +7,13 @@ The `gh` CLI preferred over the PyGithub to have an easy way to rollback bad release in command line by simple execution giving rollback commands On another hand, PyGithub is used for convenient getting commit's status from API + +To run this script on a freshly installed Ubuntu 22.04 system, it is enough to do the following commands: + +sudo apt install pip +pip install requests boto3 github PyGithub +sudo snap install gh +gh auth login """ From c7a28b137ad9ca75c44bb531fc79ba034e3e311d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 3 Apr 2024 07:24:20 +0000 Subject: [PATCH 0889/1165] Update version_date.tsv and changelogs after v24.3.2.23-lts --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v24.3.2.23-lts.md | 29 ++++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 1 + 5 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v24.3.2.23-lts.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 3daa62cb212..346868e19c4 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.3.1.2672" +ARG VERSION="24.3.2.23" ARG PACKAGES="clickhouse-keeper" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index ace01ae9a9f..36f09c092f8 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.3.1.2672" +ARG VERSION="24.3.2.23" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index e92823b686a..531a50efe96 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -27,7 +27,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="24.3.1.2672" +ARG VERSION="24.3.2.23" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v24.3.2.23-lts.md b/docs/changelogs/v24.3.2.23-lts.md new file mode 100644 index 00000000000..4d59a1cedf6 --- /dev/null +++ b/docs/changelogs/v24.3.2.23-lts.md @@ -0,0 +1,29 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.3.2.23-lts (8b7d910960c) FIXME as compared to v24.3.1.2672-lts (2c5c589a882) + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix logical error in group_by_use_nulls + grouping set + analyzer + materialize/constant [#61567](https://github.com/ClickHouse/ClickHouse/pull/61567) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix external table cannot parse data type Bool [#62115](https://github.com/ClickHouse/ClickHouse/pull/62115) ([Duc Canh Le](https://github.com/canhld94)). +* Revert "Merge pull request [#61564](https://github.com/ClickHouse/ClickHouse/issues/61564) from liuneng1994/optimize_in_single_value" [#62135](https://github.com/ClickHouse/ClickHouse/pull/62135) ([Raúl Marín](https://github.com/Algunenano)). + +#### CI Fix or Improvement (changelog entry is not required) + +* Backported in [#62030](https://github.com/ClickHouse/ClickHouse/issues/62030):. [#61869](https://github.com/ClickHouse/ClickHouse/pull/61869) ([Nikita Fomichev](https://github.com/fm4v)). +* Backported in [#62057](https://github.com/ClickHouse/ClickHouse/issues/62057): ... [#62044](https://github.com/ClickHouse/ClickHouse/pull/62044) ([Max K.](https://github.com/maxknv)). +* Backported in [#62204](https://github.com/ClickHouse/ClickHouse/issues/62204):. [#62190](https://github.com/ClickHouse/ClickHouse/pull/62190) ([Konstantin Bogdanov](https://github.com/thevar1able)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix some crashes with analyzer and group_by_use_nulls. [#61933](https://github.com/ClickHouse/ClickHouse/pull/61933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix scalars create as select [#61998](https://github.com/ClickHouse/ClickHouse/pull/61998) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Ignore IfChainToMultiIfPass if returned type changed. [#62059](https://github.com/ClickHouse/ClickHouse/pull/62059) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix type for ConvertInToEqualPass [#62066](https://github.com/ClickHouse/ClickHouse/pull/62066) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Revert output Pretty in tty [#62090](https://github.com/ClickHouse/ClickHouse/pull/62090) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index ca1a23a99db..060a0107c1e 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v24.3.2.23-lts 2024-04-03 v24.3.1.2672-lts 2024-03-27 v24.2.2.71-stable 2024-03-15 v24.2.1.2248-stable 2024-02-29 From 8f40db2fb2c520a8907914f8f5799026c43ed3f2 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Wed, 3 Apr 2024 08:26:52 +0100 Subject: [PATCH 0890/1165]
missing closing / --- docs/en/sql-reference/window-functions/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md index 19821781d0e..32ebc6d028f 100644 --- a/docs/en/sql-reference/window-functions/index.md +++ b/docs/en/sql-reference/window-functions/index.md @@ -24,7 +24,7 @@ ClickHouse supports the standard grammar for defining windows and window functio | Calculating aggregate functions over a frame (`sum(value) over (order by time)`) | ✅ (All aggregate functions are supported) | | `rank()`, `dense_rank()`, `row_number()` | ✅ | | `lag/lead(value, offset)` | ❌
You can use one of the following workarounds:
1) `any(value) over (.... rows between preceding and preceding)`, or `following` for `lead`
2) `lagInFrame/leadInFrame`, which are analogous, but respect the window frame. To get behavior identical to `lag/lead`, use `rows between unbounded preceding and unbounded following` | -| ntile(buckets) | ✅
Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). | +| ntile(buckets) | ✅
Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). | ## ClickHouse-specific Window Functions From d7827eaf57974caf9229d0bc3dcb80470a3e3fd9 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 09:48:36 +0200 Subject: [PATCH 0891/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/23104 --- .../03053_analyzer_join_alias.reference | 0 .../0_stateless/03053_analyzer_join_alias.sql | 43 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/queries/0_stateless/03053_analyzer_join_alias.reference create mode 100644 tests/queries/0_stateless/03053_analyzer_join_alias.sql diff --git a/tests/queries/0_stateless/03053_analyzer_join_alias.reference b/tests/queries/0_stateless/03053_analyzer_join_alias.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03053_analyzer_join_alias.sql b/tests/queries/0_stateless/03053_analyzer_join_alias.sql new file mode 100644 index 00000000000..7e11cc7c810 --- /dev/null +++ b/tests/queries/0_stateless/03053_analyzer_join_alias.sql @@ -0,0 +1,43 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/23104 +DROP DATABASE IF EXISTS test_03053; +CREATE DATABASE test_03053; + +CREATE TABLE test_03053.base +( +`id` UInt64, +`id2` UInt64, +`d` UInt64, +`value` UInt64 +) +ENGINE=MergeTree() +PARTITION BY d +ORDER BY (id,id2,d); + +CREATE TABLE test_03053.derived1 +( + `id1` UInt64, + `d1` UInt64, + `value1` UInt64 +) +ENGINE = MergeTree() +PARTITION BY d1 +ORDER BY (id1, d1); + +CREATE TABLE test_03053.derived2 +( + `id2` UInt64, + `d2` UInt64, + `value2` UInt64 +) +ENGINE = MergeTree() +PARTITION BY d2 +ORDER BY (id2, d2); + +SELECT + base.id AS `base.id`, + derived2.id2 AS `derived2.id2`, + derived2.value2 AS `derived2.value2`, + derived1.value1 AS `derived1.value1` +FROM test_03053.base AS base +LEFT JOIN test_03053.derived2 AS derived2 ON base.id2 = derived2.id2 +LEFT JOIN test_03053.derived1 AS derived1 ON base.id = derived1.id1; From d78ab39b4bccdd18120408ab0e050515c5fbe465 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 09:52:38 +0200 Subject: [PATCH 0892/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/21584 --- .../0_stateless/03054_analyzer_join_alias.reference | 1 + .../0_stateless/03054_analyzer_join_alias.sql | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/queries/0_stateless/03054_analyzer_join_alias.reference create mode 100644 tests/queries/0_stateless/03054_analyzer_join_alias.sql diff --git a/tests/queries/0_stateless/03054_analyzer_join_alias.reference b/tests/queries/0_stateless/03054_analyzer_join_alias.reference new file mode 100644 index 00000000000..f599e28b8ab --- /dev/null +++ b/tests/queries/0_stateless/03054_analyzer_join_alias.reference @@ -0,0 +1 @@ +10 diff --git a/tests/queries/0_stateless/03054_analyzer_join_alias.sql b/tests/queries/0_stateless/03054_analyzer_join_alias.sql new file mode 100644 index 00000000000..0bf93258aa6 --- /dev/null +++ b/tests/queries/0_stateless/03054_analyzer_join_alias.sql @@ -0,0 +1,12 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/21584 +SELECT count() +FROM +( + SELECT number AS key_1 + FROM numbers(15) +) AS x +ALL INNER JOIN +( + SELECT number AS key_1 + FROM numbers(10) +) AS z ON key_1 = z.key_1; From 6805103ce3545d3e3d94d18a2ae26d5976b2c9c6 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 09:58:11 +0200 Subject: [PATCH 0893/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/23344 --- .../03055_analyzer_subquery_group_array.reference | 1 + .../0_stateless/03055_analyzer_subquery_group_array.sql | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/03055_analyzer_subquery_group_array.reference create mode 100644 tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql diff --git a/tests/queries/0_stateless/03055_analyzer_subquery_group_array.reference b/tests/queries/0_stateless/03055_analyzer_subquery_group_array.reference new file mode 100644 index 00000000000..d05b1f927f4 --- /dev/null +++ b/tests/queries/0_stateless/03055_analyzer_subquery_group_array.reference @@ -0,0 +1 @@ +0 0 diff --git a/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql b/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql new file mode 100644 index 00000000000..071d8f8e1c8 --- /dev/null +++ b/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql @@ -0,0 +1,5 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/23344 +SELECT logTrace(repeat('Hello', 100)), ignore(*) +FROM ( + SELECT ignore((SELECT groupArrayState(([number], [number])) FROM numbers(19000))) +) From 809461ccdae2896e7b91dba36667645864a9aaae Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 10:03:47 +0200 Subject: [PATCH 0894/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/22627 --- ...3056_analyzer_double_subquery_alias.reference | 1 + .../03056_analyzer_double_subquery_alias.sql | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/03056_analyzer_double_subquery_alias.reference create mode 100644 tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql diff --git a/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.reference b/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.reference new file mode 100644 index 00000000000..72749c905a3 --- /dev/null +++ b/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.reference @@ -0,0 +1 @@ +1 1 1 diff --git a/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql b/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql new file mode 100644 index 00000000000..b86ae97c8bf --- /dev/null +++ b/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql @@ -0,0 +1,16 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/22627 +WITH + x AS + ( + SELECT 1 AS a + ), + xx AS + ( + SELECT * + FROM x + , x AS x1 + , x AS x2 + ) +SELECT * +FROM xx +WHERE a = 1; From a401d328fd35a65973306571e67539fcd8ec4909 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 10:06:11 +0200 Subject: [PATCH 0895/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/10276 --- .../03057_analyzer_subquery_alias_join.reference | 1 + .../03057_analyzer_subquery_alias_join.sql | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/queries/0_stateless/03057_analyzer_subquery_alias_join.reference create mode 100644 tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql diff --git a/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.reference b/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.reference new file mode 100644 index 00000000000..3e43e6addc0 --- /dev/null +++ b/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.reference @@ -0,0 +1 @@ +1000 100000 diff --git a/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql b/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql new file mode 100644 index 00000000000..13852471dca --- /dev/null +++ b/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql @@ -0,0 +1,12 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/10276 +SELECT + sum(x.n) as n, + sum(z.n) as n2 +FROM +( + SELECT 1000 AS n,1 as id +) AS x +join (select 10000 as n,1 as id) as y +on x.id = y.id +left join (select 100000 as n,1 as id) as z +on x.id = z.id; From 1c10c3802c156e566c7a5d131a92d3413eaa02d9 Mon Sep 17 00:00:00 2001 From: pufit Date: Wed, 3 Apr 2024 11:09:35 +0200 Subject: [PATCH 0896/1165] Docs formatting typo --- docs/en/sql-reference/functions/string-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index b4e2adbed3c..573790f7ff7 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -99,7 +99,7 @@ Alias: `OCTET_LENGTH` Returns the length of a string in Unicode code points (not: in bytes or characters). It assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined. Alias: -- `CHAR_LENGTH`` +- `CHAR_LENGTH` - `CHARACTER_LENGTH` ## leftPad From 651d382695b9399d9f74e6369772975c3b26746a Mon Sep 17 00:00:00 2001 From: Blargian Date: Wed, 3 Apr 2024 11:17:57 +0200 Subject: [PATCH 0897/1165] Minor changes to CramersV functions --- .../aggregate-functions/reference/cramersv.md | 11 ++++++++--- .../reference/cramersvbiascorrected.md | 10 ++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/cramersv.md b/docs/en/sql-reference/aggregate-functions/reference/cramersv.md index e9e2c367610..77b6043dc13 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/cramersv.md +++ b/docs/en/sql-reference/aggregate-functions/reference/cramersv.md @@ -7,21 +7,26 @@ sidebar_position: 351 [Cramer's V](https://en.wikipedia.org/wiki/Cram%C3%A9r%27s_V) (sometimes referred to as Cramer's phi) is a measure of association between two columns in a table. The result of the `cramersV` function ranges from 0 (corresponding to no association between the variables) to 1 and can reach 1 only when each value is completely determined by the other. It may be viewed as the association between two variables as a percentage of their maximum possible variation. +:::note +For a bias corrected version of Cramer's V see: [cramersVBiasCorrected](./cramersvbiascorrected.md) +::: + **Syntax** ``` sql cramersV(column1, column2) ``` -**Arguments** +**Parameters** -- `column1` and `column2` are the columns to be compared +- `column1`: first column to be compared. +- `column2`: second column to be compared. **Returned value** - a value between 0 (corresponding to no association between the columns' values) to 1 (complete association). -**Return type** is always [Float64](../../../sql-reference/data-types/float.md). +Type: always [Float64](../../../sql-reference/data-types/float.md). **Example** diff --git a/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md b/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md index f5ad3a8a937..53ec5c0a985 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md +++ b/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md @@ -5,26 +5,24 @@ sidebar_position: 352 # cramersVBiasCorrected - Cramer's V is a measure of association between two columns in a table. The result of the [`cramersV` function](./cramersv.md) ranges from 0 (corresponding to no association between the variables) to 1 and can reach 1 only when each value is completely determined by the other. The function can be heavily biased, so this version of Cramer's V uses the [bias correction](https://en.wikipedia.org/wiki/Cram%C3%A9r%27s_V#Bias_correction). - - **Syntax** ``` sql cramersVBiasCorrected(column1, column2) ``` -**Arguments** +**Parameters** -- `column1` and `column2` are the columns to be compared +- `column1`: first column to be compared. +- `column2`: second column to be compared. **Returned value** - a value between 0 (corresponding to no association between the columns' values) to 1 (complete association). -**Return type** is always [Float64](../../../sql-reference/data-types/float.md). +Type: always [Float64](../../../sql-reference/data-types/float.md). **Example** From 3e2616d921ec5c128138b57a2f20b7e07bedbf7b Mon Sep 17 00:00:00 2001 From: Blargian Date: Wed, 3 Apr 2024 11:23:42 +0200 Subject: [PATCH 0898/1165] Add 'Query:' before query for consistency --- docs/en/sql-reference/aggregate-functions/reference/cramersv.md | 2 ++ .../aggregate-functions/reference/cramersvbiascorrected.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/en/sql-reference/aggregate-functions/reference/cramersv.md b/docs/en/sql-reference/aggregate-functions/reference/cramersv.md index 77b6043dc13..2424ff95237 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/cramersv.md +++ b/docs/en/sql-reference/aggregate-functions/reference/cramersv.md @@ -32,6 +32,8 @@ Type: always [Float64](../../../sql-reference/data-types/float.md). The following two columns being compared below have no association with each other, so the result of `cramersV` is 0: +Query: + ``` sql SELECT cramersV(a, b) diff --git a/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md b/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md index 53ec5c0a985..939c04e3fdc 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md +++ b/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md @@ -28,6 +28,8 @@ Type: always [Float64](../../../sql-reference/data-types/float.md). The following two columns being compared below have a small association with each other. Notice the result of `cramersVBiasCorrected` is smaller than the result of `cramersV`: +Query: + ``` sql SELECT cramersV(a, b), From 2f45d98c970740c2263812fe3044616787f49d96 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 3 Apr 2024 10:03:04 +0000 Subject: [PATCH 0899/1165] Docs: Improve wording of DROP TABLE docs --- docs/en/sql-reference/statements/drop.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 159ab09ab94..833ff756449 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -20,11 +20,10 @@ DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] [SYNC] ## DROP TABLE -Deletes the table. -In case when `IF EMPTY` clause is specified server will check if table is empty only on replica that received initial query. +Deletes one or more tables. :::tip -Also see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md) +To undo the deletion of a table, please see see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md) ::: Syntax: @@ -33,7 +32,9 @@ Syntax: DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY] [db1.]name_1[, [db2.]name_2, ...] [ON CLUSTER cluster] [SYNC] ``` -Note that deleting multiple tables at the same time is a non-atomic deletion. If a table fails to be deleted, subsequent tables will not be deleted. +Limitations: +- If the clause `IF EMPTY` is specified, the server checks the emptiness of the table only on the replica which received the query. +- Deleting multiple tables at once is not an atomic operation, i.e. if the deletion of a table fails, subsequent tables will not be deleted. ## DROP DICTIONARY From 0d3d7b3146e1117057c60db098276d28ee3561e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 3 Apr 2024 12:12:31 +0200 Subject: [PATCH 0900/1165] Revert "Resubmit 'Update invalidate_query_response on dictionary startup'" --- src/Interpreters/ExternalLoader.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 73dacfacf79..bd56a540128 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -996,14 +996,6 @@ private: if (!new_object && !new_exception) throw Exception(ErrorCodes::LOGICAL_ERROR, "No object created and no exception raised for {}", type_name); - if (!info->object && new_object) - { - /// If we loaded the object for the first time then we should set `invalidate_query_response` to the current value. - /// Otherwise we will immediately try to reload the object again despite the fact that it was just loaded. - bool is_modified = new_object->isModified(); - LOG_TRACE(log, "Object '{}' was{} modified", name, (is_modified ? "" : " not")); - } - /// Saving the result of the loading. { LoadingGuardForAsyncLoad lock(async, mutex); From 9d9ee6ce06447bf35d2591976b43bd1e6beeed68 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 3 Apr 2024 12:16:50 +0200 Subject: [PATCH 0901/1165] Remove assertion --- src/Interpreters/Cache/FileSegment.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index f56f5d3f66c..ac42c4d75bb 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -845,7 +845,6 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) chassert(downloaded_size == range().size()); chassert(downloaded_size > 0); chassert(std::filesystem::file_size(getPath()) > 0); - chassert(queue_iterator); check_iterator(queue_iterator); } else From 4b668d09906b2052c2ee2ee043067c5075e77071 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 2 Apr 2024 10:51:41 +0200 Subject: [PATCH 0902/1165] Unlimited output_format_pretty_max_value_width for --pager Signed-off-by: Azat Khuzhin --- src/Client/ClientBase.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 7a3192d1d9c..4865ef4ef72 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -712,11 +712,20 @@ void ClientBase::adjustSettings() settings.input_format_values_allow_data_after_semicolon.changed = false; } - /// If pager is specified then output_format_pretty_max_rows is ignored, this should be handled by pager. - if (!pager.empty() && !global_context->getSettingsRef().output_format_pretty_max_rows.changed) + /// Do not limit pretty format output in case of --pager specified. + if (!pager.empty()) { - settings.output_format_pretty_max_rows = std::numeric_limits::max(); - settings.output_format_pretty_max_rows.changed = false; + if (!global_context->getSettingsRef().output_format_pretty_max_rows.changed) + { + settings.output_format_pretty_max_rows = std::numeric_limits::max(); + settings.output_format_pretty_max_rows.changed = false; + } + + if (!global_context->getSettingsRef().output_format_pretty_max_value_width.changed) + { + settings.output_format_pretty_max_value_width = std::numeric_limits::max(); + settings.output_format_pretty_max_value_width.changed = false; + } } global_context->setSettings(settings); From e19c635edd3b511adf1decffaa70829e77f59b6a Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 10:12:55 +0200 Subject: [PATCH 0903/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/4567 --- ...03058_analyzer_ambiguous_columns.reference | 0 .../03058_analyzer_ambiguous_columns.sql | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/queries/0_stateless/03058_analyzer_ambiguous_columns.reference create mode 100644 tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql diff --git a/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.reference b/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql b/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql new file mode 100644 index 00000000000..47df6e76a38 --- /dev/null +++ b/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql @@ -0,0 +1,25 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/4567 +DROP TABLE IF EXISTS fact; +DROP TABLE IF EXISTS animals; +DROP TABLE IF EXISTS colors; + +create table fact(id Int64, animal_key Int64, color_key Int64) Engine = MergeTree order by tuple(); +insert into fact values (1,1,1),(2,2,2); + +create table animals(animal_key UInt64, animal_name String) Engine = MergeTree order by tuple(); +insert into animals values (0, 'unknown'); + +create table colors(color_key UInt64, color_name String) Engine = MergeTree order by tuple(); +insert into colors values (0, 'unknown'); + + +select id, animal_name, a.animal_key, color_name, color_key +from fact a + left join (select toInt64(animal_key) animal_key, animal_name from animals) b on (a.animal_key = b.animal_key) + left join (select toInt64(color_key) color_key, color_name from colors) c on (a.color_key = c.color_key); -- { serverError AMBIGUOUS_IDENTIFIER } + +select id, animal_name, animal_key, color_name, color_key +from fact a + left join (select toInt64(animal_key) animal_key, animal_name from animals) b on (a.animal_key = b.animal_key) + left join (select toInt64(color_key) color_key, color_name from colors) c on (a.color_key = c.color_key); -- { serverError AMBIGUOUS_IDENTIFIER } + From 5e30984d8d7780182dda35c827b4c4626c06210d Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 10:19:14 +0200 Subject: [PATCH 0904/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/17710 --- .../03059_analyzer_join_engine_missing_column.reference | 0 .../03059_analyzer_join_engine_missing_column.sql | 8 ++++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.reference create mode 100644 tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql diff --git a/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.reference b/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql b/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql new file mode 100644 index 00000000000..76150335f7e --- /dev/null +++ b/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql @@ -0,0 +1,8 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/17710 +CREATE TABLE id_val(id UInt32, val UInt32) ENGINE = Memory; +CREATE TABLE id_val_join0(id UInt32, val UInt8) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls = 0; +CREATE TABLE id_val_join1(id UInt32, val UInt8) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls = 1; + +SELECT * FROM id_val ANY LEFT JOIN id_val_join0 USING (id) SETTINGS join_use_nulls = 0; + +SELECT * FROM id_val ANY LEFT JOIN id_val_join1 USING (id) SETTINGS join_use_nulls = 1; From 197e6008ea54d158b73cbc55c657f48801f08b48 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 10:24:58 +0200 Subject: [PATCH 0905/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/11068 --- .../03060_analyzer_regular_view_alias.reference | 0 .../03060_analyzer_regular_view_alias.sql | 15 +++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/queries/0_stateless/03060_analyzer_regular_view_alias.reference create mode 100644 tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql diff --git a/tests/queries/0_stateless/03060_analyzer_regular_view_alias.reference b/tests/queries/0_stateless/03060_analyzer_regular_view_alias.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql b/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql new file mode 100644 index 00000000000..ba0257d7b3b --- /dev/null +++ b/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql @@ -0,0 +1,15 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/11068 +create table vt(datetime_value DateTime, value Float64) Engine=Memory; + +create view computed_datum_hours as +SELECT + toStartOfHour(b.datetime_value) AS datetime_desc, + sum(b.value) AS value +FROM vt AS b +GROUP BY toStartOfHour(b.datetime_value); + +SELECT + toStartOfHour(b.datetime_value) AS datetime_desc, + sum(b.value) AS value +FROM vt AS b +GROUP BY toStartOfHour(b.datetime_value); From bd90cd532fdc1dff06db3a8f10dfd25c76eb4234 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 11:10:37 +0200 Subject: [PATCH 0906/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/24395 --- ...03061_analyzer_alias_as_right_key_in_join.reference | 10 ++++++++++ .../03061_analyzer_alias_as_right_key_in_join.sql | 7 +++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.reference create mode 100644 tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql diff --git a/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.reference b/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.reference new file mode 100644 index 00000000000..af98bcd6397 --- /dev/null +++ b/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.reference @@ -0,0 +1,10 @@ +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 diff --git a/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql b/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql new file mode 100644 index 00000000000..e223909a5a8 --- /dev/null +++ b/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql @@ -0,0 +1,7 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/24395 +CREATE TABLE xxxx_yyy (key UInt32, key_b ALIAS key) ENGINE=MergeTree() ORDER BY key; +INSERT INTO xxxx_yyy SELECT number FROM numbers(10); + +SELECT * +FROM xxxx_yyy AS a +INNER JOIN xxxx_yyy AS b ON a.key = b.key_b; From b66cfb4c823eae628ed8e5639d494f71768c93ea Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 11:14:18 +0200 Subject: [PATCH 0907/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/23416 --- ...062_analyzer_join_engine_missing_column.reference | 2 ++ .../03062_analyzer_join_engine_missing_column.sql | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.reference create mode 100644 tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql diff --git a/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.reference b/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.reference new file mode 100644 index 00000000000..d496ccad6b6 --- /dev/null +++ b/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.reference @@ -0,0 +1,2 @@ +abc 0 0 0 1 +abc 0 0 0 1 diff --git a/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql b/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql new file mode 100644 index 00000000000..6c24ef6f66d --- /dev/null +++ b/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql @@ -0,0 +1,12 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/23416 +create table test (TOPIC String, PARTITION UInt64, OFFSET UInt64, ID UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_03062', 'r2') ORDER BY (TOPIC, PARTITION, OFFSET); + +create table test_join (TOPIC String, PARTITION UInt64, OFFSET UInt64) ENGINE = Join(ANY, LEFT, `TOPIC`, `PARTITION`) SETTINGS join_any_take_last_row = 1; + +insert into test values('abc',0,0,0); + +insert into test_join values('abc',0,1); + +select *, joinGet('test_join', 'OFFSET', TOPIC, PARTITION) from test; + +select * from test any left join test_join using (TOPIC, PARTITION); From cc6bcb606eac18334823354500a79d2203be8c44 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 11:22:32 +0200 Subject: [PATCH 0908/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/23162 --- ...er_multi_join_wrong_table_specifier.reference | 0 ...analyzer_multi_join_wrong_table_specifier.sql | 16 ++++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.reference create mode 100644 tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql diff --git a/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.reference b/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql b/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql new file mode 100644 index 00000000000..c2c29b688cd --- /dev/null +++ b/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql @@ -0,0 +1,16 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/23162 +CREATE TABLE t1 ( k Int64, x Int64) ENGINE = Memory; + +CREATE TABLE t2( x Int64 ) ENGINE = Memory; + +create table s (k Int64, d DateTime) Engine=Memory; + +SELECT * FROM t1 +INNER JOIN s ON t1.k = s.k +INNER JOIN t2 ON t2.x = t1.x +WHERE (t1.d >= now()); -- { serverError UNKNOWN_IDENTIFIER } + +SELECT * FROM t1 +INNER JOIN s ON t1.k = s.k +WHERE (t1.d >= now()); -- { serverError UNKNOWN_IDENTIFIER } + From e09187ea2e9473cd5bad59fd5eaf61a4ddc91e30 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 11:27:43 +0200 Subject: [PATCH 0909/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/25655 --- .../0_stateless/03064_analyzer_named_subqueries.reference | 1 + .../queries/0_stateless/03064_analyzer_named_subqueries.sql | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/03064_analyzer_named_subqueries.reference create mode 100644 tests/queries/0_stateless/03064_analyzer_named_subqueries.sql diff --git a/tests/queries/0_stateless/03064_analyzer_named_subqueries.reference b/tests/queries/0_stateless/03064_analyzer_named_subqueries.reference new file mode 100644 index 00000000000..556d825db42 --- /dev/null +++ b/tests/queries/0_stateless/03064_analyzer_named_subqueries.reference @@ -0,0 +1 @@ +2 1 diff --git a/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql b/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql new file mode 100644 index 00000000000..ef8aca2fefa --- /dev/null +++ b/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql @@ -0,0 +1,5 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/25655 +SELECT + sum(t.b) / 1 a, + sum(t.a) +FROM ( SELECT 1 a, 2 b ) t; From 71d85653e9a44c340520a63374af172610a989db Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 11:29:24 +0200 Subject: [PATCH 0910/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/11757 --- .../03065_analyzer_cross_join_and_array_join.reference | 2 ++ .../0_stateless/03065_analyzer_cross_join_and_array_join.sql | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.reference create mode 100644 tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql diff --git a/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.reference b/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.reference new file mode 100644 index 00000000000..594a6a2deeb --- /dev/null +++ b/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.reference @@ -0,0 +1,2 @@ +1 3 +2 4 diff --git a/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql b/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql new file mode 100644 index 00000000000..c270a0f4504 --- /dev/null +++ b/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql @@ -0,0 +1,2 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/11757 +select * from (select [1, 2] a) aa cross join (select [3, 4] b) bb array join aa.a, bb.b; From 0758faa039edf0a1b61d05c17b2568e0a53693ea Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 11:32:26 +0200 Subject: [PATCH 0911/1165] Add test for analyzer and enable_global_with_statement=1 --- .../03066_analyzer_global_with_statement.reference | 1 + .../0_stateless/03066_analyzer_global_with_statement.sql | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/03066_analyzer_global_with_statement.reference create mode 100644 tests/queries/0_stateless/03066_analyzer_global_with_statement.sql diff --git a/tests/queries/0_stateless/03066_analyzer_global_with_statement.reference b/tests/queries/0_stateless/03066_analyzer_global_with_statement.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03066_analyzer_global_with_statement.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql b/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql new file mode 100644 index 00000000000..338eb30e6ff --- /dev/null +++ b/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql @@ -0,0 +1,7 @@ +WITH 0 AS test +SELECT * +FROM +( + SELECT 1 AS test +) +SETTINGS enable_global_with_statement = 1 From 7872d920586e7843a8dd1db027ecf7c9fa75b07c Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 11:35:08 +0200 Subject: [PATCH 0912/1165] Add test for analyzer and complex alias join and with --- .../03067_analyzer_complex_alias_join.reference | 1 + .../0_stateless/03067_analyzer_complex_alias_join.sql | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03067_analyzer_complex_alias_join.reference create mode 100644 tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql diff --git a/tests/queries/0_stateless/03067_analyzer_complex_alias_join.reference b/tests/queries/0_stateless/03067_analyzer_complex_alias_join.reference new file mode 100644 index 00000000000..6192a595f1e --- /dev/null +++ b/tests/queries/0_stateless/03067_analyzer_complex_alias_join.reference @@ -0,0 +1 @@ +key \N diff --git a/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql b/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql new file mode 100644 index 00000000000..7d1264a6116 --- /dev/null +++ b/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql @@ -0,0 +1,9 @@ +with d as (select 'key'::Varchar(255) c, 'x'::Varchar(255) s) +SELECT r1, c as r2 +FROM ( + SELECT t as s, c as r1 + FROM ( SELECT 'y'::Varchar(255) as t, 'x'::Varchar(255) as s) t1 + LEFT JOIN d USING (s) + ) t2 +LEFT JOIN d using (s) +SETTINGS join_use_nulls=1; From ece7099017285db81637c5f65fad9678301a22f9 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 12:04:36 +0200 Subject: [PATCH 0913/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/6571 --- .../03068_analyzer_distributed_join.reference | 2 + .../03068_analyzer_distributed_join.sql | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tests/queries/0_stateless/03068_analyzer_distributed_join.reference create mode 100644 tests/queries/0_stateless/03068_analyzer_distributed_join.sql diff --git a/tests/queries/0_stateless/03068_analyzer_distributed_join.reference b/tests/queries/0_stateless/03068_analyzer_distributed_join.reference new file mode 100644 index 00000000000..1444d39d957 --- /dev/null +++ b/tests/queries/0_stateless/03068_analyzer_distributed_join.reference @@ -0,0 +1,2 @@ +localhost 9000 0 0 0 +localhost 9000 0 0 0 diff --git a/tests/queries/0_stateless/03068_analyzer_distributed_join.sql b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql new file mode 100644 index 00000000000..714a64a01b6 --- /dev/null +++ b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql @@ -0,0 +1,55 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/6571 +CREATE TABLE LINEITEM_shard ON CLUSTER test_shard_localhost +( + L_ORDERKEY UInt64, + L_COMMITDATE UInt32, + L_RECEIPTDATE UInt32 +) +ENGINE = MergeTree() +ORDER BY L_ORDERKEY; + +CREATE TABLE LINEITEM AS LINEITEM_shard +ENGINE = Distributed('test_shard_localhost', currentDatabase(), LINEITEM_shard, rand()); + +CREATE TABLE ORDERS_shard ON CLUSTER test_shard_localhost +( + O_ORDERKEY UInt64, + O_ORDERPRIORITY UInt32 +) +ENGINE = MergeTree() +ORDER BY O_ORDERKEY; + +CREATE TABLE ORDERS AS ORDERS_shard +ENGINE = Distributed('test_shard_localhost', currentDatabase(), ORDERS_shard, rand()); + +SET joined_subquery_requires_alias=0; + +select + O_ORDERPRIORITY, + count(*) as order_count +from ORDERS JOIN ( + select L_ORDERKEY + from + LINEITEM_shard + group by L_ORDERKEY + having any(L_COMMITDATE < L_RECEIPTDATE) +) on O_ORDERKEY=L_ORDERKEY +group by O_ORDERPRIORITY +order by O_ORDERPRIORITY +limit 1; + +SET joined_subquery_requires_alias=1; + +select + O_ORDERPRIORITY, + count(*) as order_count +from ORDERS JOIN ( + select L_ORDERKEY + from + LINEITEM_shard + group by L_ORDERKEY + having any(L_COMMITDATE < L_RECEIPTDATE) +) AS x on O_ORDERKEY=L_ORDERKEY +group by O_ORDERPRIORITY +order by O_ORDERPRIORITY +limit 1; From 71450c909a30e2ae4120aad5dbff518f6d5985da Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 3 Apr 2024 10:39:43 +0000 Subject: [PATCH 0914/1165] Fix lambda(tuple(x), x + 1) syntax in analyzer --- src/Analyzer/QueryTreeBuilder.cpp | 2 +- src/Interpreters/ActionsVisitor.cpp | 7 +++-- src/Parsers/ASTFunction.cpp | 11 ++++++++ src/Parsers/ASTFunction.h | 3 +++ src/Parsers/ExpressionElementParsers.cpp | 4 +-- .../02343_analyzer_lambdas.reference | 8 ++++++ .../0_stateless/02343_analyzer_lambdas.sql | 26 +++++++++++++++++++ 7 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index df80f46b3cd..d2587d74b7c 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -558,7 +558,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co } else if (const auto * function = expression->as()) { - if (function->is_lambda_function) + if (function->is_lambda_function || isASTLambdaFunction(*function)) { const auto & lambda_arguments_and_expression = function->arguments->as().children; auto & lambda_arguments_tuple = lambda_arguments_and_expression.at(0)->as(); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 16e2449206d..093c266c785 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -1130,12 +1130,11 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & const auto * query_parameter = child->as(); if (function && function->name == "lambda") { + if (!isASTLambdaFunction(*function)) + throw Exception(ErrorCodes::SYNTAX_ERROR, "Lambda function definition expects two arguments, first argument must be a tuple of arguments"); + /// If the argument is a lambda expression, just remember its approximate type. - if (function->arguments->children.size() != 2) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "lambda requires two arguments"); - const auto * lambda_args_tuple = function->arguments->children.at(0)->as(); - if (!lambda_args_tuple || lambda_args_tuple->name != "tuple") throw Exception(ErrorCodes::TYPE_MISMATCH, "First argument of lambda must be a tuple"); diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 07eea86ef81..cdc9a471e98 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -793,4 +793,15 @@ bool tryGetFunctionNameInto(const IAST * ast, String & name) return false; } +bool isASTLambdaFunction(const ASTFunction & function) +{ + if (function.name == "lambda" && function.arguments && function.arguments->children.size() == 2) + { + const auto * lambda_args_tuple = function.arguments->children.at(0)->as(); + return lambda_args_tuple && lambda_args_tuple->name == "tuple"; + } + + return false; +} + } diff --git a/src/Parsers/ASTFunction.h b/src/Parsers/ASTFunction.h index 631b6285bfa..3a94691f25d 100644 --- a/src/Parsers/ASTFunction.h +++ b/src/Parsers/ASTFunction.h @@ -111,4 +111,7 @@ inline String getFunctionName(const ASTPtr & ast) { return getFunctionName(ast.g inline std::optional tryGetFunctionName(const ASTPtr & ast) { return tryGetFunctionName(ast.get()); } inline bool tryGetFunctionNameInto(const ASTPtr & ast, String & name) { return tryGetFunctionNameInto(ast.get(), name); } +/// Checks if function is a lambda function definition `lambda((x, y), x + y)` +bool isASTLambdaFunction(const ASTFunction & function); + } diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 67f4a306292..29b49736783 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1545,8 +1545,8 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e { if (auto * func = lambda->as(); func && func->name == "lambda") { - if (func->arguments->children.size() != 2) - throw Exception(ErrorCodes::SYNTAX_ERROR, "lambda requires two arguments"); + if (!isASTLambdaFunction(*func)) + throw Exception(ErrorCodes::SYNTAX_ERROR, "Lambda function definition expects two arguments, first argument must be a tuple of arguments"); const auto * lambda_args_tuple = func->arguments->children.at(0)->as(); if (!lambda_args_tuple || lambda_args_tuple->name != "tuple") diff --git a/tests/queries/0_stateless/02343_analyzer_lambdas.reference b/tests/queries/0_stateless/02343_analyzer_lambdas.reference index 8d29481c255..62d9e9f4726 100644 --- a/tests/queries/0_stateless/02343_analyzer_lambdas.reference +++ b/tests/queries/0_stateless/02343_analyzer_lambdas.reference @@ -27,3 +27,11 @@ Lambda untuple Lambda carrying 2 1 1 0 +Lambda legacy syntax +[2,3,4] +[2,3,4] +[2,3,4] +['hello','world'] +[2,3,4] +[2,3,4] 2 +[2,3,4] 2 1 diff --git a/tests/queries/0_stateless/02343_analyzer_lambdas.sql b/tests/queries/0_stateless/02343_analyzer_lambdas.sql index b90f7b32b57..0c257cf6f18 100644 --- a/tests/queries/0_stateless/02343_analyzer_lambdas.sql +++ b/tests/queries/0_stateless/02343_analyzer_lambdas.sql @@ -65,5 +65,31 @@ SELECT 'Lambda carrying'; WITH (functor, x) -> functor(x) AS lambda, x -> x + 1 AS functor_1, x -> toString(x) AS functor_2 SELECT lambda(functor_1, 1), lambda(functor_2, 1); WITH (functor, x) -> functor(x) AS lambda, x -> x + 1 AS functor_1, x -> toString(x) AS functor_2 SELECT lambda(functor_1, id), lambda(functor_2, id) FROM test_table; + +SELECT 'Lambda legacy syntax'; + +SELECT arrayMap(lambda(tuple(x), x + 1), [1, 2, 3]); + +WITH 222 AS lambda +SELECT arrayMap(lambda(tuple(x), x + 1), [1, 2, 3]); + +SELECT arrayMap(lambda((x,), x + 1), [1, 2, 3]); + +SELECT arraySort(lambda((x, y), y), ['world', 'hello'], [2, 1]); + +WITH 222 AS lambda +SELECT arrayMap(lambda((x, ), x + 1), [1, 2, 3]); + +WITH x -> x + 1 AS lambda +SELECT arrayMap(lambda(tuple(x), x + 1), [1, 2, 3]), lambda(1); + +-- lambda(tuple(x), x + 1) parsed as lambda definion but not as call of lambda defined in WITH +WITH (x, y) -> y AS lambda +SELECT arrayMap(lambda(tuple(x), x + 1), [1, 2, 3]), lambda(tuple(x), x + 1), 1 AS x; -- { serverError BAD_ARGUMENTS } + +WITH (x, y) -> y AS lambda2 +SELECT arrayMap(lambda(tuple(x), x + 1), [1, 2, 3]), lambda2(tuple(x), x + 1), 1 AS x; + + DROP TABLE test_table_tuple; DROP TABLE test_table; From e693418f6312772bfcde777533f141754c377d41 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 3 Apr 2024 10:45:45 +0000 Subject: [PATCH 0915/1165] Fix crash in index definition containing sql udf --- .../InterpreterCreateIndexQuery.cpp | 4 +- .../MySQL/InterpretersMySQLDDLQuery.cpp | 6 +- src/Parsers/ASTIndexDeclaration.cpp | 60 +++++++++++++++---- src/Parsers/ASTIndexDeclaration.h | 15 ++--- src/Parsers/ParserCreateIndexQuery.cpp | 11 ++-- src/Parsers/ParserCreateQuery.cpp | 10 ++-- src/Storages/IndicesDescription.cpp | 18 +++--- src/Storages/MergeTree/MergeTreeData.cpp | 5 +- .../StorageSystemDataSkippingIndices.cpp | 6 +- ...033_index_definition_sql_udf_bug.reference | 1 + .../03033_index_definition_sql_udf_bug.sql | 21 +++++++ 11 files changed, 106 insertions(+), 51 deletions(-) create mode 100644 tests/queries/0_stateless/03033_index_definition_sql_udf_bug.reference create mode 100644 tests/queries/0_stateless/03033_index_definition_sql_udf_bug.sql diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index aed4b0587b4..6045b5d2e24 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -39,12 +39,12 @@ BlockIO InterpreterCreateIndexQuery::execute() } // Noop if allow_create_index_without_type = true. throw otherwise - if (!create_index.index_decl->as()->type) + if (!create_index.index_decl->as()->getType()) { if (!current_context->getSettingsRef().allow_create_index_without_type) { throw Exception(ErrorCodes::INCORRECT_QUERY, "CREATE INDEX without TYPE is forbidden." - " SET allow_create_index_without_type=1 to ignore this statements."); + " SET allow_create_index_without_type=1 to ignore this statements"); } else { diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index fd7ffca2872..4821d607d0e 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -498,14 +498,12 @@ ASTs InterpreterCreateImpl::getRewrittenQueries( columns->columns->children.emplace_back(create_materialized_column_declaration(version_column_name, "UInt64", UInt64(1))); /// Add minmax skipping index for _version column. - auto version_index = std::make_shared(); - version_index->name = version_column_name; auto index_expr = std::make_shared(version_column_name); auto index_type = makeASTFunction("minmax"); index_type->no_empty_args = true; - version_index->set(version_index->expr, index_expr); - version_index->set(version_index->type, index_type); + auto version_index = std::make_shared(index_expr, index_type, version_column_name); version_index->granularity = 1; + ASTPtr indices = std::make_shared(); indices->children.push_back(version_index); columns->set(columns->indices, indices); diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index 8dac5389c80..32689f31546 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -8,24 +8,57 @@ namespace DB { -ASTPtr ASTIndexDeclaration::clone() const +namespace ErrorCodes { - auto res = std::make_shared(); - - res->name = name; - if (granularity) - res->granularity = granularity; - if (expr) - res->set(res->expr, expr->clone()); - if (type) - res->set(res->type, type->clone()); - return res; + extern const int LOGICAL_ERROR; } +ASTIndexDeclaration::ASTIndexDeclaration(ASTPtr expression, ASTPtr type, const String & name_) + : name(name_) +{ + if (!expression) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Index declaration must have an expression"); + children.push_back(expression); + + if (type) + { + if (!dynamic_cast(type.get())) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Index declaration type must be a function"); + children.push_back(type); + } +} + +ASTPtr ASTIndexDeclaration::clone() const +{ + auto expr = getExpression(); + auto type = getType(); + auto res = std::make_shared(expr, type, name); + res->granularity = granularity; + + return res; +} + +ASTPtr ASTIndexDeclaration::getExpression() const +{ + if (children.size() <= expression_idx) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Index declaration must have an expression"); + return children[expression_idx]; +} + +std::shared_ptr ASTIndexDeclaration::getType() const +{ + if (children.size() <= type_idx) + return nullptr; + auto func_ast = std::dynamic_pointer_cast(children[type_idx]); + if (!func_ast) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Index declaration type must be a function"); + return func_ast; +} + void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { - if (expr) + if (auto expr = getExpression()) { if (part_of_create_index_query) { @@ -46,11 +79,12 @@ void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & sta } } - if (type) + if (auto type = getType()) { s.ostr << (s.hilite ? hilite_keyword : "") << " TYPE " << (s.hilite ? hilite_none : ""); type->formatImpl(s, state, frame); } + if (granularity) { s.ostr << (s.hilite ? hilite_keyword : "") << " GRANULARITY " << (s.hilite ? hilite_none : ""); diff --git a/src/Parsers/ASTIndexDeclaration.h b/src/Parsers/ASTIndexDeclaration.h index 1fbf5e12695..dd05ad08184 100644 --- a/src/Parsers/ASTIndexDeclaration.h +++ b/src/Parsers/ASTIndexDeclaration.h @@ -16,9 +16,9 @@ public: static const auto DEFAULT_ANNOY_INDEX_GRANULARITY = 100'000'000uz; static const auto DEFAULT_USEARCH_INDEX_GRANULARITY = 100'000'000uz; + ASTIndexDeclaration(ASTPtr expression, ASTPtr type, const String & name_); + String name; - IAST * expr; - ASTFunction * type; UInt64 granularity; bool part_of_create_index_query = false; @@ -28,11 +28,12 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; - void forEachPointerToChild(std::function f) override - { - f(reinterpret_cast(&expr)); - f(reinterpret_cast(&type)); - } + ASTPtr getExpression() const; + std::shared_ptr getType() const; + +private: + static constexpr size_t expression_idx = 0; + static constexpr size_t type_idx = 1; }; } diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp index 3b1b9d8ec84..fd2bbbab177 100644 --- a/src/Parsers/ParserCreateIndexQuery.cpp +++ b/src/Parsers/ParserCreateIndexQuery.cpp @@ -54,19 +54,18 @@ bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected return false; } - auto index = std::make_shared(); + /// name is set below in ParserCreateIndexQuery + auto index = std::make_shared(expr, type, ""); index->part_of_create_index_query = true; - index->set(index->expr, expr); - if (type) - index->set(index->type, type); if (granularity) index->granularity = granularity->as().value.safeGet(); else { - if (index->type && index->type->name == "annoy") + auto index_type = index->getType(); + if (index_type && index_type->name == "annoy") index->granularity = ASTIndexDeclaration::DEFAULT_ANNOY_INDEX_GRANULARITY; - else if (index->type && index->type->name == "usearch") + else if (index_type && index_type->name == "usearch") index->granularity = ASTIndexDeclaration::DEFAULT_USEARCH_INDEX_GRANULARITY; else index->granularity = ASTIndexDeclaration::DEFAULT_INDEX_GRANULARITY; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 1510cc8e195..ff88b58760b 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -205,18 +205,16 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return false; } - auto index = std::make_shared(); - index->name = name->as().name(); - index->set(index->expr, expr); - index->set(index->type, type); + auto index = std::make_shared(expr, type, name->as().name()); if (granularity) index->granularity = granularity->as().value.safeGet(); else { - if (index->type->name == "annoy") + auto index_type = index->getType(); + if (index_type->name == "annoy") index->granularity = ASTIndexDeclaration::DEFAULT_ANNOY_INDEX_GRANULARITY; - else if (index->type->name == "usearch") + else if (index_type->name == "usearch") index->granularity = ASTIndexDeclaration::DEFAULT_USEARCH_INDEX_GRANULARITY; else index->granularity = ASTIndexDeclaration::DEFAULT_INDEX_GRANULARITY; diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 14555dca63b..13375a1f4dc 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -85,22 +85,23 @@ IndexDescription IndexDescription::getIndexFromAST(const ASTPtr & definition_ast if (index_definition->name.empty()) throw Exception(ErrorCodes::INCORRECT_QUERY, "Skip index must have name in definition."); - if (!index_definition->type) + auto index_type = index_definition->getType(); + if (!index_type) throw Exception(ErrorCodes::INCORRECT_QUERY, "TYPE is required for index"); - if (index_definition->type->parameters && !index_definition->type->parameters->children.empty()) + if (index_type->parameters && !index_type->parameters->children.empty()) throw Exception(ErrorCodes::INCORRECT_QUERY, "Index type cannot have parameters"); IndexDescription result; result.definition_ast = index_definition->clone(); result.name = index_definition->name; - result.type = Poco::toLower(index_definition->type->name); + result.type = Poco::toLower(index_definition->getType()->name); result.granularity = index_definition->granularity; ASTPtr expr_list; - if (index_definition->expr) + if (auto index_expression = index_definition->getExpression()) { - expr_list = extractKeyExpressionList(index_definition->expr->clone()); + expr_list = extractKeyExpressionList(index_expression); ReplaceAliasToExprVisitor::Data data{columns}; ReplaceAliasToExprVisitor{data}.visit(expr_list); @@ -125,12 +126,11 @@ IndexDescription IndexDescription::getIndexFromAST(const ASTPtr & definition_ast result.data_types.push_back(elem.type); } - const auto & definition_arguments = index_definition->type->arguments; - if (definition_arguments) + if (index_type && index_type->arguments) { - for (size_t i = 0; i < definition_arguments->children.size(); ++i) + for (size_t i = 0; i < index_type->arguments->children.size(); ++i) { - const auto * argument = definition_arguments->children[i]->as(); + const auto * argument = index_type->arguments->children[i]->as(); if (!argument) throw Exception(ErrorCodes::INCORRECT_QUERY, "Only literals can be skip index arguments"); result.arguments.emplace_back(argument->value); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e984f306e2e..6163a8dbee9 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -652,8 +652,9 @@ void MergeTreeData::checkProperties( if (!allow_suspicious_indices && !attach) { const auto * index_ast = typeid_cast(index.definition_ast.get()); - if (const auto * index_function = typeid_cast(index_ast->expr)) - checkSuspiciousIndices(index_function); + auto index_function = index_ast ? index_ast->getType() : nullptr; + if (index_function) + checkSuspiciousIndices(index_function.get()); } MergeTreeIndexFactory::instance().validate(index, attach); diff --git a/src/Storages/System/StorageSystemDataSkippingIndices.cpp b/src/Storages/System/StorageSystemDataSkippingIndices.cpp index ff782647c79..2afc03d0e5e 100644 --- a/src/Storages/System/StorageSystemDataSkippingIndices.cpp +++ b/src/Storages/System/StorageSystemDataSkippingIndices.cpp @@ -131,8 +131,10 @@ protected: // 'type_full' column if (column_mask[src_index++]) { - if (auto * expression = index.definition_ast->as(); expression && expression->type) - res_columns[res_index++]->insert(queryToString(*expression->type)); + auto * expression = index.definition_ast->as(); + auto index_type = expression ? expression->getType() : nullptr; + if (index_type) + res_columns[res_index++]->insert(queryToString(*index_type)); else res_columns[res_index++]->insertDefault(); } diff --git a/tests/queries/0_stateless/03033_index_definition_sql_udf_bug.reference b/tests/queries/0_stateless/03033_index_definition_sql_udf_bug.reference new file mode 100644 index 00000000000..5782593a455 --- /dev/null +++ b/tests/queries/0_stateless/03033_index_definition_sql_udf_bug.reference @@ -0,0 +1 @@ +2 2 2 diff --git a/tests/queries/0_stateless/03033_index_definition_sql_udf_bug.sql b/tests/queries/0_stateless/03033_index_definition_sql_udf_bug.sql new file mode 100644 index 00000000000..84ab1d33c94 --- /dev/null +++ b/tests/queries/0_stateless/03033_index_definition_sql_udf_bug.sql @@ -0,0 +1,21 @@ +-- Tags: no-parallel + +DROP FUNCTION IF EXISTS test_func_1; +CREATE FUNCTION test_func_1 AS (a, b, c) -> ((a + b) + c); + +DROP TABLE IF EXISTS t4_2; +CREATE TABLE t4_2 +( + `col1` Int64 NOT NULL COMMENT 'test', + `col2` Float64 NOT NULL, + `col3` Int64 NOT NULL, + INDEX ind4 test_func_1(col1, col3, col1) TYPE set(51) GRANULARITY 5 +) +ENGINE = MergeTree +ORDER BY col1 +; + +INSERT INTO t4_2 (col1, col2, col3) SELECT number, number, number FROM numbers(10); + +SELECT * FROM t4_2 WHERE test_func_1(col1, col3, col1) = 6 +SETTINGS force_data_skipping_indices = 'ind4'; From fdb752de6f56a4cfdc2b48b342c873b861d2ddb9 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 12:52:28 +0200 Subject: [PATCH 0916/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/4432 --- .../03069_analyzer_with_alias_in_array_join.reference | 2 ++ .../0_stateless/03069_analyzer_with_alias_in_array_join.sql | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.reference create mode 100644 tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql diff --git a/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.reference b/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.reference new file mode 100644 index 00000000000..1191247b6d9 --- /dev/null +++ b/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.reference @@ -0,0 +1,2 @@ +1 +2 diff --git a/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql b/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql new file mode 100644 index 00000000000..84ad0b4e199 --- /dev/null +++ b/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql @@ -0,0 +1,5 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/4432 +WITH [1, 2] AS zz +SELECT x +FROM system.one +ARRAY JOIN zz AS x From 0782ccaa91fa4a850cea00f52a660ee818e8e3c8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 3 Apr 2024 12:53:09 +0200 Subject: [PATCH 0917/1165] Update docs/en/sql-reference/statements/drop.md Co-authored-by: Han Fei --- docs/en/sql-reference/statements/drop.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 833ff756449..98b849ecf3b 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -23,7 +23,7 @@ DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] [SYNC] Deletes one or more tables. :::tip -To undo the deletion of a table, please see see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md) +To undo the deletion of a table, please see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md) ::: Syntax: From c9430180258631d44fb7677c1ea725db586d63e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 3 Apr 2024 13:00:25 +0200 Subject: [PATCH 0918/1165] Include table name in paranoid checks --- src/Storages/StorageReplicatedMergeTree.cpp | 32 +++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8ca061db4ec..70d77432847 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1524,8 +1524,13 @@ void StorageReplicatedMergeTree::paranoidCheckForCoveredPartsInZooKeeperOnStart( if (!found) { - LOG_WARNING(log, "Part {} exists in ZooKeeper and covered by another part in ZooKeeper ({}), but doesn't exist on any disk. " - "It may cause false-positive 'part is lost forever' messages", part_name, covering_part); + LOG_WARNING( + log, + "Part {} of table {} exists in ZooKeeper and covered by another part in ZooKeeper ({}), but doesn't exist on any disk. " + "It may cause false-positive 'part is lost forever' messages", + part_name, + getStorageID().getNameForLogs(), + covering_part); ProfileEvents::increment(ProfileEvents::ReplicatedCoveredPartsInZooKeeperOnStart); chassert(false); } @@ -2351,8 +2356,12 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::executeFetchShared } } -static void paranoidCheckForCoveredPartsInZooKeeper(const ZooKeeperPtr & zookeeper, const String & replica_path, - MergeTreeDataFormatVersion format_version, const String & covering_part_name) +static void paranoidCheckForCoveredPartsInZooKeeper( + const ZooKeeperPtr & zookeeper, + const String & replica_path, + MergeTreeDataFormatVersion format_version, + const String & covering_part_name, + const StorageReplicatedMergeTree & storage) { #ifdef ABORT_ON_LOGICAL_ERROR constexpr bool paranoid_check_for_covered_parts_default = true; @@ -2371,8 +2380,12 @@ static void paranoidCheckForCoveredPartsInZooKeeper(const ZooKeeperPtr & zookeep { auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); if (drop_range_info.contains(part_info)) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Part {} remains in ZooKeeper after DROP_RANGE {}", part_name, covering_part_name); + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Part {} from table {} remains in ZooKeeper after DROP_RANGE {}", + part_name, + storage.getStorageID().getNameForLogs(), + covering_part_name); } } @@ -2434,7 +2447,7 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) /// Forcibly remove parts from ZooKeeper removePartsFromZooKeeperWithRetries(parts_to_remove); - paranoidCheckForCoveredPartsInZooKeeper(getZooKeeper(), replica_path, format_version, entry.new_part_name); + paranoidCheckForCoveredPartsInZooKeeper(getZooKeeper(), replica_path, format_version, entry.new_part_name, *this); if (entry.detach) LOG_DEBUG(log, "Detached {} parts inside {}.", parts_to_remove.size(), entry.new_part_name); @@ -2572,7 +2585,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) LOG_INFO(log, "All parts from REPLACE PARTITION command have been already attached"); removePartsFromZooKeeperWithRetries(parts_to_remove); if (replace) - paranoidCheckForCoveredPartsInZooKeeper(getZooKeeper(), replica_path, format_version, entry_replace.drop_range_part_name); + paranoidCheckForCoveredPartsInZooKeeper( + getZooKeeper(), replica_path, format_version, entry_replace.drop_range_part_name, *this); return true; } @@ -2893,7 +2907,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) removePartsFromZooKeeperWithRetries(parts_to_remove); if (replace) - paranoidCheckForCoveredPartsInZooKeeper(getZooKeeper(), replica_path, format_version, entry_replace.drop_range_part_name); + paranoidCheckForCoveredPartsInZooKeeper(getZooKeeper(), replica_path, format_version, entry_replace.drop_range_part_name, *this); res_parts.clear(); parts_to_remove.clear(); cleanup_thread.wakeup(); From 0b831ab87a1d6cc510042d021639548b2600bfdb Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 13:09:43 +0200 Subject: [PATCH 0919/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/8259 --- ...0_analyzer_CTE_scalar_as_numbers.reference | 25 +++++++++++++++++++ .../03070_analyzer_CTE_scalar_as_numbers.sql | 5 ++++ 2 files changed, 30 insertions(+) create mode 100644 tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.reference create mode 100644 tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql diff --git a/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.reference b/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.reference new file mode 100644 index 00000000000..595b4d6b5b8 --- /dev/null +++ b/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.reference @@ -0,0 +1,25 @@ +0 25 +1 25 +2 25 +3 25 +4 25 +5 25 +6 25 +7 25 +8 25 +9 25 +10 25 +11 25 +12 25 +13 25 +14 25 +15 25 +16 25 +17 25 +18 25 +19 25 +20 25 +21 25 +22 25 +23 25 +24 25 diff --git a/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql b/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql new file mode 100644 index 00000000000..672c4f53e5f --- /dev/null +++ b/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql @@ -0,0 +1,5 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/8259 +with + (select 25) as something +select *, something +from numbers(toUInt64(assumeNotNull(something))); From f822791ee397d6def5bd64adc47b077b484d0058 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 13:17:19 +0200 Subject: [PATCH 0920/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/9233 --- ...r_array_join_forbid_non_existing_columns.reference | 0 ...nalyzer_array_join_forbid_non_existing_columns.sql | 11 +++++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.reference create mode 100644 tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql diff --git a/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.reference b/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql b/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql new file mode 100644 index 00000000000..af81e3c2819 --- /dev/null +++ b/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql @@ -0,0 +1,11 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/9233 +SELECT * +FROM +( + SELECT + [1, 2, 3] AS x, + [4, 5, 6] AS y +) +ARRAY JOIN + x, + Y; -- { serverError UNKNOWN_IDENTIFIER } From 15dd5ce5f665467d33bb02e7d7ba2decfade3e87 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 3 Apr 2024 08:39:16 -0300 Subject: [PATCH 0921/1165] use raw literal string --- src/IO/S3/URI.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 69b539cde8b..7f628d27f34 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -40,7 +40,7 @@ URI::URI(const std::string & uri_) /// Case when AWS Private Link Interface is being used /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html - static const RE2 aws_private_link_style_pattern("bucket\\.vpce\\-([a-z0-9\\-.:]+)\\.vpce.amazonaws.com"); + static const RE2 aws_private_link_style_pattern(R"(bucket\.vpce\-([a-z0-9\-.:]+)\.vpce.amazonaws.com)"); /// Case when bucket name and key represented in path of S3 URL. /// E.g. (https://s3.region.amazonaws.com/bucket-name/key) From 40817de19a674763af7f7b83bd7712053f0ae18d Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 13:44:31 +0200 Subject: [PATCH 0922/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/14699 --- .../03072_analyzer_missing_columns_from_subquery.reference | 1 + .../03072_analyzer_missing_columns_from_subquery.sql | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.reference create mode 100644 tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql diff --git a/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.reference b/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql b/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql new file mode 100644 index 00000000000..68ff81413b7 --- /dev/null +++ b/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql @@ -0,0 +1,2 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/14699 +select * from (select number from numbers(1)) where not ignore(*); From 44b3ce9ec88ac147e33856170c402bac3444bb54 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Wed, 3 Apr 2024 19:49:55 +0800 Subject: [PATCH 0923/1165] review fix --- src/Core/SettingsChangesHistory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index f578e0c8d0a..0ecd6d81f25 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -125,7 +125,7 @@ static std::map sett {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."}, {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."}, {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."}, - {"input_format_hive_text_allow_variable_number_of_columns", true, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."}, + {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, From 04445b30ef3722f601dde6825fbffd1a9704fbc1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 3 Apr 2024 11:50:14 +0000 Subject: [PATCH 0924/1165] Fix another logical error in group_by_use_nulls. --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 4 ++-- .../0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f5474ddb662..837d309d031 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -6155,7 +6155,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id return resolved_expression_it->second; } - bool is_nullable_group_by_key = scope.nullable_group_by_keys.contains(node) && !scope.expressions_in_resolve_process_stack.hasAggregateFunction(); + bool is_nullable_group_by_key = scope.nullable_group_by_keys.contains(node); if (is_nullable_group_by_key) ++scope.found_nullable_group_by_key_in_scope; @@ -6452,7 +6452,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id validateTreeSize(node, scope.context->getSettingsRef().max_expanded_ast_elements, node_to_tree_size); - if (is_nullable_group_by_key && scope.found_nullable_group_by_key_in_scope == 1) + if (is_nullable_group_by_key && scope.found_nullable_group_by_key_in_scope == 1 && !scope.expressions_in_resolve_process_stack.hasAggregateFunction()) { node = node->clone(); node->convertToNullable(); diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql index 012da547581..28042fc5b44 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql @@ -5,3 +5,5 @@ SELECT tuple(tuple(number)) as x FROM numbers(10) GROUP BY (number, tuple(number select tuple(array(number)) as x FROM numbers(10) GROUP BY number, array(number) WITH ROLLUP order by x; SELECT tuple(number) AS x FROM numbers(10) GROUP BY GROUPING SETS (number) order by x; + +SELECT ignore(toFixedString('Lambda as function parameter', 28), toNullable(28), ignore(8)), sum(marks) FROM system.parts GROUP BY GROUPING SETS ((2)) FORMAT Null settings optimize_injective_functions_in_group_by=1, optimize_group_by_function_keys=1, group_by_use_nulls=1; -- { serverError ILLEGAL_AGGREGATION } From c576884a0b40d4870a7cbb08e62ebffec24ee328 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 13:51:06 +0200 Subject: [PATCH 0925/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/27068 --- .../03073_analyzer_alias_as_column_name.reference | 2 ++ .../0_stateless/03073_analyzer_alias_as_column_name.sql | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03073_analyzer_alias_as_column_name.reference create mode 100644 tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql diff --git a/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.reference b/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.reference new file mode 100644 index 00000000000..ca6280d2dbf --- /dev/null +++ b/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.reference @@ -0,0 +1,2 @@ +1 1997-02-01 +2 1997-02-01 diff --git a/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql b/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql new file mode 100644 index 00000000000..0e7d2eb95c7 --- /dev/null +++ b/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql @@ -0,0 +1,8 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/27068 +CREATE TABLE test ( id String, create_time DateTime ) ENGINE = MergeTree ORDER BY id; + +insert into test values(1,'1970-02-01 00:00:00'); +insert into test values(2,'1970-02-01 00:00:00'); +insert into test values(3,'1970-03-01 00:00:00'); + +select id,'1997-02-01' as create_time from test where test.create_time='1970-02-01 00:00:00' ORDER BY id From 2c569b0d4645c1945bc420fb70160c331d81e51b Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 3 Apr 2024 11:13:25 +0000 Subject: [PATCH 0926/1165] fix --- src/Parsers/ASTIndexDeclaration.cpp | 10 ++++++++-- src/Storages/IndicesDescription.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 7 ++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index 32689f31546..0c36644356f 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -31,8 +31,14 @@ ASTIndexDeclaration::ASTIndexDeclaration(ASTPtr expression, ASTPtr type, const S ASTPtr ASTIndexDeclaration::clone() const { - auto expr = getExpression(); - auto type = getType(); + ASTPtr expr = getExpression(); + if (expr) + expr = expr->clone(); + + ASTPtr type = getType(); + if (type) + type = type->clone(); + auto res = std::make_shared(expr, type, name); res->granularity = granularity; diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 13375a1f4dc..cef8fd85f97 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -95,7 +95,7 @@ IndexDescription IndexDescription::getIndexFromAST(const ASTPtr & definition_ast IndexDescription result; result.definition_ast = index_definition->clone(); result.name = index_definition->name; - result.type = Poco::toLower(index_definition->getType()->name); + result.type = Poco::toLower(index_type->name); result.granularity = index_definition->granularity; ASTPtr expr_list; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6163a8dbee9..b60bd365eb2 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -652,9 +652,10 @@ void MergeTreeData::checkProperties( if (!allow_suspicious_indices && !attach) { const auto * index_ast = typeid_cast(index.definition_ast.get()); - auto index_function = index_ast ? index_ast->getType() : nullptr; - if (index_function) - checkSuspiciousIndices(index_function.get()); + ASTPtr index_expression = index_ast ? index_ast->getExpression() : nullptr; + const auto * index_expression_ptr = index_expression ? typeid_cast(index_expression.get()) : nullptr; + if (index_expression_ptr) + checkSuspiciousIndices(index_expression_ptr); } MergeTreeIndexFactory::instance().validate(index, attach); From 1c98c4752e916e077a1938c663324709fe5ec1c0 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 13:55:38 +0200 Subject: [PATCH 0927/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/28687 --- .../03074_analyzer_alias_column_in_view.reference | 2 ++ .../0_stateless/03074_analyzer_alias_column_in_view.sql | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/03074_analyzer_alias_column_in_view.reference create mode 100644 tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql diff --git a/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.reference b/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.reference new file mode 100644 index 00000000000..aa47d0d46d4 --- /dev/null +++ b/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql b/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql new file mode 100644 index 00000000000..6f970421788 --- /dev/null +++ b/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql @@ -0,0 +1,6 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/28687 +create view alias (dummy int, n alias dummy) as select * from system.one; + +select n from alias; + +select * from alias where n=0; From f2997c9c8920ffeb569fcf72e4a5739597868b80 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 3 Apr 2024 12:08:25 +0000 Subject: [PATCH 0928/1165] Fix settings changes history --- src/Core/SettingsChangesHistory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 219b444be2f..ac68155a0af 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -86,10 +86,10 @@ namespace SettingsChangesHistory static std::map settings_changes_history = { {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, + {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"}, }}, {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, - {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"}, {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"}, {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"}, {"page_cache_inject_eviction", false, false, "Added userspace page cache"}, From 90ac11171c80f18914e0f246fb886c6c5d9c32ca Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 3 Apr 2024 12:18:54 +0000 Subject: [PATCH 0929/1165] Fix crash --- src/Interpreters/RewriteOrderByVisitor.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Interpreters/RewriteOrderByVisitor.cpp b/src/Interpreters/RewriteOrderByVisitor.cpp index 694dec84b7a..26817b70dc0 100644 --- a/src/Interpreters/RewriteOrderByVisitor.cpp +++ b/src/Interpreters/RewriteOrderByVisitor.cpp @@ -39,9 +39,8 @@ void RewriteOrderBy::visit(ASTPtr & ast, Data &) { // clone w/o children auto clone = std::make_shared(*order_by_elem); - clone->children.clear(); - clone->children.emplace_back(identifier); + clone->children[0] = identifier; new_order_by->children.emplace_back(clone); } if (!new_order_by->children.empty()) From f2d78f650dbb0c2145f16cd0337835bfbdc879df Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 3 Apr 2024 14:32:19 +0200 Subject: [PATCH 0930/1165] Fix global trace collector --- src/Common/ThreadStatus.h | 4 ---- src/Coordination/Standalone/ThreadStatusExt.cpp | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 550cb76e736..02bf82e2da3 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -308,11 +308,7 @@ public: void flushUntrackedMemory(); -#ifdef CLICKHOUSE_KEEPER_STANDALONE_BUILD - void initGlobalProfiler(UInt64, UInt64) {} -#else void initGlobalProfiler(UInt64 global_profiler_real_time_period, UInt64 global_profiler_cpu_time_period); -#endif private: void applyGlobalSettings(); diff --git a/src/Coordination/Standalone/ThreadStatusExt.cpp b/src/Coordination/Standalone/ThreadStatusExt.cpp index 97f7287be8c..2b89e2f024d 100644 --- a/src/Coordination/Standalone/ThreadStatusExt.cpp +++ b/src/Coordination/Standalone/ThreadStatusExt.cpp @@ -11,4 +11,8 @@ void CurrentThread::attachToGroup(const ThreadGroupPtr &) { } +void ThreadStatus::initGlobalProfiler(UInt64, UInt64) +{ +} + } From ef0a9e889f2c9c3c9a1952e10b29fb1e94613ca3 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 14:35:34 +0200 Subject: [PATCH 0931/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/28777 --- .../03075_analyzer_subquery_alias.reference | 1 + .../0_stateless/03075_analyzer_subquery_alias.sql | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/queries/0_stateless/03075_analyzer_subquery_alias.reference create mode 100644 tests/queries/0_stateless/03075_analyzer_subquery_alias.sql diff --git a/tests/queries/0_stateless/03075_analyzer_subquery_alias.reference b/tests/queries/0_stateless/03075_analyzer_subquery_alias.reference new file mode 100644 index 00000000000..556d825db42 --- /dev/null +++ b/tests/queries/0_stateless/03075_analyzer_subquery_alias.reference @@ -0,0 +1 @@ +2 1 diff --git a/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql b/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql new file mode 100644 index 00000000000..897b189b133 --- /dev/null +++ b/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql @@ -0,0 +1,10 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/28777 +SELECT + sum(q0.a2) AS a1, + sum(q0.a1) AS a9 +FROM +( + SELECT + 1 AS a1, + 2 AS a2 +) AS q0; From 8ae1f4c4b316e6f81e87ab70e7f1d11ad7e0e771 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 14:37:58 +0200 Subject: [PATCH 0932/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/29734 --- ...76_analyzer_multiple_joins_alias.reference | 0 .../03076_analyzer_multiple_joins_alias.sql | 51 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.reference create mode 100644 tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql diff --git a/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.reference b/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql b/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql new file mode 100644 index 00000000000..8b8b76a5be1 --- /dev/null +++ b/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql @@ -0,0 +1,51 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/29734 +SELECT * +FROM +( + SELECT 1 AS x +) AS a +INNER JOIN +( + SELECT + 1 AS x, + 2 AS y +) AS b ON (a.x = b.x) AND (a.y = b.y); -- { serverError UNKNOWN_IDENTIFIER } + + + +SELECT * +FROM +( + SELECT 1 AS x +) AS a +INNER JOIN +( + SELECT + 1 AS x, + 2 AS y +) AS b ON (a.x = b.x) AND (a.y = b.y) +INNER JOIN +( + SELECT 3 AS x +) AS c ON a.x = c.x; -- { serverError UNKNOWN_IDENTIFIER } + + +SELECT * +FROM +( + SELECT number AS x + FROM numbers(10) +) AS a +INNER JOIN +( + SELECT + number AS x, + number AS y + FROM numbers(10) +) AS b ON (a.x = b.x) AND (a.y = b.y) +INNER JOIN +( + SELECT number AS x + FROM numbers(10) +) AS c ON a.x = c.x; -- { serverError UNKNOWN_IDENTIFIER } + From c3f1130de86a94decff8521dbfb30c20c6a984e5 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 15:10:27 +0200 Subject: [PATCH 0933/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/33825 --- ...er_multi_scalar_subquery_aliases.reference | 21 ++++++++++++++++++ ...analyzer_multi_scalar_subquery_aliases.sql | 22 +++++++++++++++++++ ...er_multi_scalar_subquery_aliases.reference | 21 ++++++++++++++++++ ...analyzer_multi_scalar_subquery_aliases.sql | 16 ++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.reference create mode 100644 tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql create mode 100644 tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.reference create mode 100644 tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql diff --git a/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.reference b/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.reference new file mode 100644 index 00000000000..6b134f711d4 --- /dev/null +++ b/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.reference @@ -0,0 +1,21 @@ +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +1 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 +9 9 9 9 diff --git a/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql b/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql new file mode 100644 index 00000000000..3d558bdd602 --- /dev/null +++ b/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql @@ -0,0 +1,22 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/33825 +CREATE TABLE t1 (i Int64, j Int64) ENGINE = Memory; +INSERT INTO t1 SELECT number, number FROM system.numbers LIMIT 10; +SELECT + (SELECT max(i) FROM t1) as i, + (SELECT max(i) FROM t1) as j, + (SELECT max(i) FROM t1) as k, + (SELECT max(i) FROM t1) as l +FROM t1; + +SELECT 1; + +WITH ( + SELECT max(i) + FROM t1 + ) AS value +SELECT + value AS i, + value AS j, + value AS k, + value AS l +FROM t1; diff --git a/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.reference b/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.reference new file mode 100644 index 00000000000..b2c49b655d4 --- /dev/null +++ b/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.reference @@ -0,0 +1,21 @@ +111111111111 +111111111111 +111111111111 +111111111111 +111111111111 +111111111111 +111111111111 +111111111111 +111111111111 +111111111111 +1 +2222222222 +2222222222 +2222222222 +2222222222 +2222222222 +2222222222 +2222222222 +2222222222 +2222222222 +2222222222 diff --git a/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql b/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql new file mode 100644 index 00000000000..ded6bfbe4e3 --- /dev/null +++ b/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql @@ -0,0 +1,16 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/33825 +CREATE TABLE t2 (first_column Int64, second_column Int64) ENGINE = Memory; +INSERT INTO t2 SELECT number, number FROM system.numbers LIMIT 10; + + +SELECT ( + SELECT 111111111111 + ) AS first_column +FROM t2; + +SELECT 1; + +SELECT ( + SELECT 2222222222 + ) AS second_column +FROM t2; From f6a240b7be2982b4625ffdc21e1ceeee4cd68859 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 3 Apr 2024 10:11:56 -0300 Subject: [PATCH 0934/1165] add some unit tests --- src/IO/S3/URI.h | 1 + src/IO/tests/gtest_s3_uri.cpp | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h index 2873728bc78..79f3da3fbbb 100644 --- a/src/IO/S3/URI.h +++ b/src/IO/S3/URI.h @@ -17,6 +17,7 @@ namespace DB::S3 * The following patterns are allowed: * s3://bucket/key * http(s)://endpoint/bucket/key + * TODO specify aws private link */ struct URI { diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index 5bf0dfb962d..b3ceb875362 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -74,6 +74,26 @@ const TestCase TestCases[] = { "data", "", true}, + {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/root/nested/file.txt"), + "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com", + "root", + "nested/file.txt", + "", + true}, + // Test with a file with no extension + {S3::URI("https://bucket.vpce-03b2c987f1bd55c5f-j3b4vg7w.s3.ap-southeast-2.vpce.amazonaws.com/some_bucket/document"), + "https://bucket.vpce-03b2c987f1bd55c5f-j3b4vg7w.s3.ap-southeast-2.vpce.amazonaws.com", + "some_bucket", + "document", + "", + true}, + // Test with a deeply nested file path + {S3::URI("https://bucket.vpce-0242cd56f1bd55c5f-l5b7vg8x.s3.sa-east-1.vpce.amazonaws.com/some_bucket/b/c/d/e/f/g/h/i/j/data.json"), + "https://bucket.vpce-0242cd56f1bd55c5f-l5b7vg8x.s3.sa-east-1.vpce.amazonaws.com", + "some_bucket", + "b/c/d/e/f/g/h/i/j/data.json", + "", + true}, }; class S3UriTest : public testing::TestWithParam From aaf9bb0e581ed16ac9a4a35dd49487ec12f30992 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 3 Apr 2024 13:12:05 +0000 Subject: [PATCH 0935/1165] Fixing NULL random seed for generateRandom with analyzer. --- src/Storages/checkAndGetLiteralArgument.cpp | 11 +++++++++-- src/TableFunctions/TableFunctionGenerateRandom.cpp | 13 +++++++++++-- .../0_stateless/01087_table_function_generate.sql | 2 ++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/Storages/checkAndGetLiteralArgument.cpp b/src/Storages/checkAndGetLiteralArgument.cpp index 5baf47fe91a..39dc27cd5e8 100644 --- a/src/Storages/checkAndGetLiteralArgument.cpp +++ b/src/Storages/checkAndGetLiteralArgument.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { @@ -12,8 +13,14 @@ namespace ErrorCodes template T checkAndGetLiteralArgument(const ASTPtr & arg, const String & arg_name) { - if (arg && arg->as()) - return checkAndGetLiteralArgument(*arg->as(), arg_name); + if (arg) + { + if (const auto * func = arg->as(); func && func->name == "_CAST") + return checkAndGetLiteralArgument(func->arguments->children.at(0), arg_name); + + if (arg->as()) + return checkAndGetLiteralArgument(*arg->as(), arg_name); + } throw Exception( ErrorCodes::BAD_ARGUMENTS, diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp index af284594987..15772562079 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -88,7 +89,11 @@ void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, Co // All the arguments must be literals. for (const auto & arg : args) { - if (!arg->as()) + const IAST * arg_raw = arg.get(); + if (const auto * func = arg_raw->as(); func && func->name == "_CAST") + arg_raw = func->arguments->children.at(0).get(); + + if (!arg_raw->as()) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "All arguments of table function '{}' except structure argument must be literals. " @@ -107,7 +112,11 @@ void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, Co if (args.size() >= arg_index + 1) { - const auto & literal = args[arg_index]->as(); + const IAST * arg_raw = args[arg_index].get(); + if (const auto * func = arg_raw->as(); func && func->name == "_CAST") + arg_raw = func->arguments->children.at(0).get(); + + const auto & literal = arg_raw->as(); ++arg_index; if (!literal.value.isNull()) random_seed = checkAndGetLiteralArgument(literal, "random_seed"); diff --git a/tests/queries/0_stateless/01087_table_function_generate.sql b/tests/queries/0_stateless/01087_table_function_generate.sql index 10657dbd63a..ff7c3f3477d 100644 --- a/tests/queries/0_stateless/01087_table_function_generate.sql +++ b/tests/queries/0_stateless/01087_table_function_generate.sql @@ -195,3 +195,5 @@ SELECT a, b, c, d, e, f, g, hex(h) FROM test_table_2 ORDER BY a, b, c, d, e, f, SELECT '-'; DROP TABLE IF EXISTS test_table_2; + +select * from generateRandom('x UInt64', Null, 10, 2) limit 2 format Null; From 8e6cbc8b31c93e3825219cc47463c0e854b0a26d Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 3 Apr 2024 15:13:59 +0200 Subject: [PATCH 0936/1165] several fixes for client's keep alive connections --- .../Net/include/Poco/Net/HTTPClientSession.h | 13 + base/poco/Net/include/Poco/Net/HTTPMessage.h | 3 + .../Net/include/Poco/Net/HTTPServerParams.h | 2 +- .../Net/include/Poco/Net/HTTPServerSession.h | 2 + base/poco/Net/src/HTTPClientSession.cpp | 39 +- base/poco/Net/src/HTTPMessage.cpp | 41 ++ base/poco/Net/src/HTTPServerConnection.cpp | 13 +- base/poco/Net/src/HTTPServerSession.cpp | 6 + src/Common/HTTPConnectionPool.cpp | 94 ++-- src/Common/tests/gtest_connection_pool.cpp | 423 ++++++++++++++---- src/Core/ServerSettings.h | 4 +- src/IO/ConnectionTimeouts.cpp | 6 +- 12 files changed, 502 insertions(+), 144 deletions(-) diff --git a/base/poco/Net/include/Poco/Net/HTTPClientSession.h b/base/poco/Net/include/Poco/Net/HTTPClientSession.h index 1cef988566c..b418937c4d5 100644 --- a/base/poco/Net/include/Poco/Net/HTTPClientSession.h +++ b/base/poco/Net/include/Poco/Net/HTTPClientSession.h @@ -213,6 +213,13 @@ namespace Net Poco::Timespan getKeepAliveTimeout() const; /// Returns the connection timeout for HTTP connections. + bool isKeepAliveExpired(double reliability = 1.0) const; + /// Returns if the connection is expired with some margin as fraction of timeout as reliability + + double getKeepAliveReliability() const; + /// Returns the current fraction of keep alive timeout when connection is considered safe to use + /// It helps to avoid situation when a client uses nearly expired connection and receives NoMessageException + virtual std::ostream & sendRequest(HTTPRequest & request); /// Sends the header for the given HTTP request to /// the server. @@ -361,6 +368,7 @@ namespace Net Poco::SharedPtr _pRequestStream; Poco::SharedPtr _pResponseStream; + static const double _defaultKeepAliveReliabilityLevel; static ProxyConfig _globalProxyConfig; HTTPClientSession(const HTTPClientSession &); @@ -455,6 +463,11 @@ namespace Net _lastRequest = time; } + inline double HTTPClientSession::getKeepAliveReliability() const + { + return _defaultKeepAliveReliabilityLevel; + } + } } // namespace Poco::Net diff --git a/base/poco/Net/include/Poco/Net/HTTPMessage.h b/base/poco/Net/include/Poco/Net/HTTPMessage.h index 0bef50803a8..994807ffbff 100644 --- a/base/poco/Net/include/Poco/Net/HTTPMessage.h +++ b/base/poco/Net/include/Poco/Net/HTTPMessage.h @@ -120,6 +120,9 @@ namespace Net /// The value is set to "Keep-Alive" if keepAlive is /// true, or to "Close" otherwise. + void setKeepAliveTimeout(int timeout); + int getKeepAliveTimeout() const; + bool getKeepAlive() const; /// Returns true if /// * the message has a Connection header field and its value is "Keep-Alive" diff --git a/base/poco/Net/include/Poco/Net/HTTPServerParams.h b/base/poco/Net/include/Poco/Net/HTTPServerParams.h index 3c836a630a0..d614c62d57a 100644 --- a/base/poco/Net/include/Poco/Net/HTTPServerParams.h +++ b/base/poco/Net/include/Poco/Net/HTTPServerParams.h @@ -44,7 +44,7 @@ namespace Net /// - timeout: 60 seconds /// - keepAlive: true /// - maxKeepAliveRequests: 0 - /// - keepAliveTimeout: 10 seconds + /// - keepAliveTimeout: 15 seconds void setServerName(const std::string & serverName); /// Sets the name and port (name:port) that the server uses to identify itself. diff --git a/base/poco/Net/include/Poco/Net/HTTPServerSession.h b/base/poco/Net/include/Poco/Net/HTTPServerSession.h index ec928af304f..3df7995509a 100644 --- a/base/poco/Net/include/Poco/Net/HTTPServerSession.h +++ b/base/poco/Net/include/Poco/Net/HTTPServerSession.h @@ -56,6 +56,8 @@ namespace Net SocketAddress serverAddress(); /// Returns the server's address. + void setKeepAliveTimeout(Poco::Timespan keepAliveTimeout); + private: bool _firstRequest; Poco::Timespan _keepAliveTimeout; diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp index 33a3dcc4901..59800232ba9 100644 --- a/base/poco/Net/src/HTTPClientSession.cpp +++ b/base/poco/Net/src/HTTPClientSession.cpp @@ -37,6 +37,7 @@ namespace Net { HTTPClientSession::ProxyConfig HTTPClientSession::_globalProxyConfig; +const double HTTPClientSession::_defaultKeepAliveReliabilityLevel = 0.9; HTTPClientSession::HTTPClientSession(): @@ -220,7 +221,11 @@ void HTTPClientSession::setGlobalProxyConfig(const ProxyConfig& config) void HTTPClientSession::setKeepAliveTimeout(const Poco::Timespan& timeout) { - _keepAliveTimeout = timeout; + if (connected()) + { + throw Poco::IllegalStateException("cannot change keep alive timeout on initiated connection"); + } + _keepAliveTimeout = timeout; } @@ -243,6 +248,8 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request) reconnect(); if (!keepAlive) request.setKeepAlive(false); + if (keepAlive && !request.has(HTTPMessage::CONNECTION_KEEP_ALIVE) && _keepAliveTimeout.totalSeconds() > 0) + request.setKeepAliveTimeout(_keepAliveTimeout.totalSeconds()); if (!request.has(HTTPRequest::HOST) && !_host.empty()) request.setHost(_host, _port); if (!_proxyConfig.host.empty() && !bypassProxy()) @@ -324,6 +331,14 @@ std::istream& HTTPClientSession::receiveResponse(HTTPResponse& response) _mustReconnect = getKeepAlive() && !response.getKeepAlive(); + if (!_mustReconnect) + { + /// when server sends its keep alive timeout, client has to follow that value + auto timeout = response.getKeepAliveTimeout(); + if (timeout > 0) + _keepAliveTimeout = Poco::Timespan(timeout, 0); + } + if (!_expectResponseBody || response.getStatus() < 200 || response.getStatus() == HTTPResponse::HTTP_NO_CONTENT || response.getStatus() == HTTPResponse::HTTP_NOT_MODIFIED) _pResponseStream = new HTTPFixedLengthInputStream(*this, 0); else if (response.getChunkedTransferEncoding()) @@ -430,15 +445,17 @@ std::string HTTPClientSession::proxyRequestPrefix() const return result; } +bool HTTPClientSession::isKeepAliveExpired(double reliability) const +{ + Poco::Timestamp now; + return Timespan(Timestamp::TimeDiff(reliability *_keepAliveTimeout.totalMicroseconds())) <= now - _lastRequest; +} bool HTTPClientSession::mustReconnect() const { if (!_mustReconnect) - { - Poco::Timestamp now; - return _keepAliveTimeout <= now - _lastRequest; - } - else return true; + return isKeepAliveExpired(_defaultKeepAliveReliabilityLevel); + return true; } @@ -511,14 +528,16 @@ void HTTPClientSession::assign(Poco::Net::HTTPClientSession & session) if (buffered()) throw Poco::LogicException("assign to a session with not empty buffered data"); - attachSocket(session.detachSocket()); setLastRequest(session.getLastRequest()); setResolvedHost(session.getResolvedHost()); - setKeepAlive(session.getKeepAlive()); + setProxyConfig(session.getProxyConfig()); setTimeout(session.getConnectionTimeout(), session.getSendTimeout(), session.getReceiveTimeout()); - setKeepAliveTimeout(session.getKeepAliveTimeout()); - setProxyConfig(session.getProxyConfig()); + setKeepAlive(session.getKeepAlive()); + if (!connected()) + setKeepAliveTimeout(session.getKeepAliveTimeout()); + + attachSocket(session.detachSocket()); session.reset(); } diff --git a/base/poco/Net/src/HTTPMessage.cpp b/base/poco/Net/src/HTTPMessage.cpp index 0cd234ee9cb..2f974b8bf0b 100644 --- a/base/poco/Net/src/HTTPMessage.cpp +++ b/base/poco/Net/src/HTTPMessage.cpp @@ -17,6 +17,7 @@ #include "Poco/NumberFormatter.h" #include "Poco/NumberParser.h" #include "Poco/String.h" +#include using Poco::NumberFormatter; @@ -179,4 +180,44 @@ bool HTTPMessage::getKeepAlive() const } +void HTTPMessage::setKeepAliveTimeout(int timeout) +{ + add(HTTPMessage::CONNECTION_KEEP_ALIVE, std::format("timeout={}", timeout)); +} + + +int parseTimeoutFromHeaderValue(const std::string_view header_value) +{ + static const std::string_view timeout_param = "timeout="; + + auto timeout_pos = header_value.find(timeout_param); + if (timeout_pos == std::string::npos) + timeout_pos = header_value.size(); + if (timeout_pos != header_value.size()) + timeout_pos += timeout_param.size(); + + auto timeout_end = header_value.find(',', timeout_pos); + if (timeout_end == std::string::npos) + timeout_end = header_value.size(); + + auto timeout_value_substr = header_value.substr(timeout_pos, timeout_end - timeout_pos); + if (timeout_value_substr.empty()) + return -1; + + int value = 0; + auto [ptr, ec] = std::from_chars(timeout_value_substr.begin(), timeout_value_substr.end(), value); + + if (ec == std::errc()) + return value; + + return -1; +} + + +int HTTPMessage::getKeepAliveTimeout() const +{ + const std::string& ka_header = get(HTTPMessage::CONNECTION_KEEP_ALIVE, HTTPMessage::EMPTY); + return parseTimeoutFromHeaderValue(ka_header); +} + } } // namespace Poco::Net diff --git a/base/poco/Net/src/HTTPServerConnection.cpp b/base/poco/Net/src/HTTPServerConnection.cpp index c57984b0162..d5eb29d3134 100644 --- a/base/poco/Net/src/HTTPServerConnection.cpp +++ b/base/poco/Net/src/HTTPServerConnection.cpp @@ -88,7 +88,18 @@ void HTTPServerConnection::run() pHandler->handleRequest(request, response); session.setKeepAlive(_pParams->getKeepAlive() && response.getKeepAlive() && session.canKeepAlive()); - } + + /// all that fuzz is all about to make session close with less timeout than 15s (set in HTTPServerParams c-tor) + if (_pParams->getKeepAlive() && response.getKeepAlive() && session.canKeepAlive()) + { + int value = response.getKeepAliveTimeout(); + if (value < 0) + value = request.getKeepAliveTimeout(); + if (value > 0) + session.setKeepAliveTimeout(Poco::Timespan(value, 0)); + } + + } else sendErrorResponse(session, HTTPResponse::HTTP_NOT_IMPLEMENTED); } catch (Poco::Exception&) diff --git a/base/poco/Net/src/HTTPServerSession.cpp b/base/poco/Net/src/HTTPServerSession.cpp index d4f2b24879e..f67a63a9e0e 100644 --- a/base/poco/Net/src/HTTPServerSession.cpp +++ b/base/poco/Net/src/HTTPServerSession.cpp @@ -33,6 +33,12 @@ HTTPServerSession::~HTTPServerSession() { } +void HTTPServerSession::setKeepAliveTimeout(Poco::Timespan keepAliveTimeout) +{ + _keepAliveTimeout = keepAliveTimeout; +} + + bool HTTPServerSession::hasMoreRequests() { diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index cd2505df7f3..21165bbc62d 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -83,17 +84,15 @@ namespace } - size_t roundUp(size_t x, size_t rounding) + constexpr size_t roundUp(size_t x, size_t rounding) { chassert(rounding > 0); - return (x + (rounding - 1)) / rounding * rounding; - } - - - Poco::Timespan divide(const Poco::Timespan span, int divisor) - { - return Poco::Timespan(Poco::Timestamp::TimeDiff(span.totalMicroseconds() / divisor)); + return (x + rounding) / rounding * rounding; } + static_assert(roundUp(10000, 100) == 10100); + static_assert(roundUp(10001, 100) == 10100); + static_assert(roundUp(10099, 100) == 10100); + static_assert(roundUp(10100, 100) == 10200); } namespace DB @@ -202,8 +201,9 @@ public: if (total_connections_in_group >= limits.warning_limit && total_connections_in_group >= mute_warning_until) { - LOG_WARNING(log, "Too many active sessions in group {}, count {}, warning limit {}", type, total_connections_in_group, limits.warning_limit); mute_warning_until = roundUp(total_connections_in_group, limits.warning_step); + LOG_WARNING(log, "Too many active sessions in group {}, count {}, warning limit {}, next warning at {}", + type, total_connections_in_group, limits.warning_limit, mute_warning_until); } } @@ -213,7 +213,7 @@ public: --total_connections_in_group; - const size_t reduced_warning_limit = limits.warning_limit > 10 ? limits.warning_limit - 10 : 1; + const size_t reduced_warning_limit = limits.warning_limit > 10 ? limits.warning_limit - 20 : 1; if (mute_warning_until > 0 && total_connections_in_group < reduced_warning_limit) { LOG_WARNING(log, "Sessions count is OK in the group {}, count {}", type, total_connections_in_group); @@ -221,6 +221,12 @@ public: } } + void atPoolDestroy(size_t connections) + { + std::lock_guard lock(mutex); + total_connections_in_group -= connections; + } + HTTPConnectionGroupType getType() const { return type; } const IHTTPConnectionPoolForEndpoint::Metrics & getMetrics() const { return metrics; } @@ -273,9 +279,15 @@ private: public: using Ptr = std::shared_ptr; + using Session::mustReconnect; + + void markAsExpired() + { + isExpired = true; + } + void reconnect() override { - ProfileEvents::increment(metrics.reset); Session::close(); if (auto lock = pool.lock()) @@ -352,6 +364,11 @@ private: std::istream & result = Session::receiveResponse(response); result.exceptions(std::ios::badbit); + // that line is for temporary debug, will be removed + if (response.has(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE)) + LOG_WARNING(log, "received keep alive header: {}", + response.get(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE, Poco::Net::HTTPMessage::EMPTY)); + response_stream = &result; response_stream_completed = false; @@ -392,10 +409,11 @@ private: } response_stream = nullptr; - if (auto lock = pool.lock()) - lock->atConnectionDestroy(*this); - else - ProfileEvents::increment(metrics.reset); + group->atConnectionDestroy(); + + if (!isExpired) + if (auto lock = pool.lock()) + lock->atConnectionDestroy(*this); CurrentMetrics::sub(metrics.active_count); } @@ -404,10 +422,11 @@ private: friend class EndpointConnectionPool; template - explicit PooledConnection(EndpointConnectionPool::WeakPtr pool_, IHTTPConnectionPoolForEndpoint::Metrics metrics_, Args &&... args) - : Session(args...), pool(std::move(pool_)), metrics(std::move(metrics_)) + explicit PooledConnection(EndpointConnectionPool::WeakPtr pool_, ConnectionGroup::Ptr group_, IHTTPConnectionPoolForEndpoint::Metrics metrics_, Args &&... args) + : Session(args...), pool(std::move(pool_)), group(group_), metrics(std::move(metrics_)) { CurrentMetrics::add(metrics.active_count); + group->atConnectionCreate(); } template @@ -433,10 +452,12 @@ private: return request_stream_completed && response_stream_completed; } - WeakPtr pool; + EndpointConnectionPool::WeakPtr pool; + ConnectionGroup::Ptr group; IHTTPConnectionPoolForEndpoint::Metrics metrics; + bool isExpired = false; - Poco::Logger * log = &Poco::Logger::get("PooledConnection"); + LoggerPtr log = getLogger("PooledConnection"); std::ostream * request_stream = nullptr; std::istream * response_stream = nullptr; @@ -484,7 +505,6 @@ public: IHTTPConnectionPoolForEndpoint::ConnectionPtr getConnection(const ConnectionTimeouts & timeouts) override { - Poco::Timestamp now; std::vector expired_connections; SCOPE_EXIT({ @@ -494,8 +514,9 @@ public: { std::lock_guard lock(mutex); + expired_connections.reserve(stored_connections.size()); - wipeExpiredImpl(expired_connections, now); + wipeExpiredImpl(expired_connections); if (!stored_connections.empty()) { @@ -526,7 +547,6 @@ public: size_t wipeExpired() override { - Poco::Timestamp now; std::vector expired_connections; SCOPE_EXIT({ @@ -535,19 +555,21 @@ public: }); std::lock_guard lock(mutex); - return wipeExpiredImpl(expired_connections, now); + return wipeExpiredImpl(expired_connections); } - size_t wipeExpiredImpl(std::vector & expired_connections, Poco::Timestamp now) TSA_REQUIRES(mutex) + size_t wipeExpiredImpl(std::vector & expired_connections) TSA_REQUIRES(mutex) { + auto isSoftLimitReached = group->isSoftLimitReached(); while (!stored_connections.empty()) { auto connection = stored_connections.top(); - if (!isExpired(now, connection)) + if (!isExpired(connection, isSoftLimitReached)) return stored_connections.size(); stored_connections.pop(); + connection->markAsExpired(); expired_connections.push_back(connection); } @@ -569,16 +591,16 @@ private: WeakPtr getWeakFromThis() { return EndpointConnectionPool::weak_from_this(); } - bool isExpired(Poco::Timestamp & now, ConnectionPtr connection) + bool isExpired(ConnectionPtr connection, bool isSoftLimitReached) TSA_REQUIRES(mutex) { - if (group->isSoftLimitReached()) - return now > (connection->getLastRequest() + divide(connection->getKeepAliveTimeout(), 10)); - return now > connection->getLastRequest() + connection->getKeepAliveTimeout(); + if (isSoftLimitReached) + return connection->isKeepAliveExpired(0.1); + return connection->isKeepAliveExpired(0.8); } ConnectionPtr allocateNewConnection() { - ConnectionPtr connection = PooledConnection::create(this->getWeakFromThis(), getMetrics(), host, port); + ConnectionPtr connection = PooledConnection::create(this->getWeakFromThis(), group, getMetrics(), host, port); connection->setKeepAlive(true); if (!proxy_configuration.isEmpty()) @@ -586,8 +608,6 @@ private: connection->setProxyConfig(proxyConfigurationToPocoProxyConfig(proxy_configuration)); } - group->atConnectionCreate(); - return connection; } @@ -619,8 +639,6 @@ private: void atConnectionDestroy(PooledConnection & connection) { - group->atConnectionDestroy(); - if (!connection.connected() || connection.mustReconnect() || !connection.isCompleted() || connection.buffered() || group->isStoreLimitReached()) { @@ -631,14 +649,14 @@ private: auto connection_to_store = allocateNewConnection(); connection_to_store->assign(connection); - CurrentMetrics::add(getMetrics().stored_count, 1); - ProfileEvents::increment(getMetrics().preserved, 1); - { MemoryTrackerSwitcher switcher{&total_memory_tracker}; std::lock_guard lock(mutex); stored_connections.push(connection_to_store); } + + CurrentMetrics::add(getMetrics().stored_count, 1); + ProfileEvents::increment(getMetrics().preserved, 1); } @@ -726,7 +744,7 @@ createConnectionPool(ConnectionGroup::Ptr group, std::string host, UInt16 port, class HTTPConnectionPools::Impl { private: - const size_t DEFAULT_WIPE_TIMEOUT_SECONDS = 5 * 60; + const size_t DEFAULT_WIPE_TIMEOUT_SECONDS = 10 * 60; const Poco::Timespan wipe_timeout = Poco::Timespan(DEFAULT_WIPE_TIMEOUT_SECONDS, 0); ConnectionGroup::Ptr disk_group = std::make_shared(HTTPConnectionGroupType::DISK); diff --git a/src/Common/tests/gtest_connection_pool.cpp b/src/Common/tests/gtest_connection_pool.cpp index dcc3c11fd52..36bf8bc7dae 100644 --- a/src/Common/tests/gtest_connection_pool.cpp +++ b/src/Common/tests/gtest_connection_pool.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -17,6 +16,39 @@ namespace { +template +class SafeHandler +{ +public: + using Ptr = std::shared_ptr>; + + SafeHandler() = default; + SafeHandler(SafeHandler&) = delete; + SafeHandler& operator=(SafeHandler&) = delete; + + T get() + { + std::lock_guard lock(mutex); + return obj; + } + + void set(T && options_) + { + std::lock_guard lock(mutex); + obj = std::move(options_); + } + +protected: + std::mutex mutex; + T obj = {}; +}; + +struct RequestOptions +{ + size_t slowdown_receive = 0; + int overwrite_keep_alive_timeout = 0; +}; + size_t stream_copy_n(std::istream & in, std::ostream & out, std::size_t count = std::numeric_limits::max()) { const size_t buffer_size = 4096; @@ -47,13 +79,19 @@ size_t stream_copy_n(std::istream & in, std::ostream & out, std::size_t count = class MockRequestHandler : public Poco::Net::HTTPRequestHandler { public: - explicit MockRequestHandler(std::shared_ptr> slowdown_) - : slowdown(std::move(slowdown_)) + explicit MockRequestHandler(SafeHandler::Ptr options_) + : options(options_) { } void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override { + int value = request.getKeepAliveTimeout(); + ASSERT_GT(value, 0); + + if (options->get().overwrite_keep_alive_timeout > 0) + response.setKeepAliveTimeout(options->get().overwrite_keep_alive_timeout); + response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); auto size = request.getContentLength(); if (size > 0) @@ -61,28 +99,29 @@ public: else response.setChunkedTransferEncoding(true); // or chunk encoding - sleepForSeconds(*slowdown); + if (options->get().slowdown_receive > 0) + sleepForSeconds(options->get().slowdown_receive); stream_copy_n(request.stream(), response.send(), size); } - std::shared_ptr> slowdown; + SafeHandler::Ptr options; }; class HTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory { public: - explicit HTTPRequestHandlerFactory(std::shared_ptr> slowdown_) - : slowdown(std::move(slowdown_)) + explicit HTTPRequestHandlerFactory(SafeHandler::Ptr options_) + : options(options_) { } Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override { - return new MockRequestHandler(slowdown); + return new MockRequestHandler(options); } - std::shared_ptr> slowdown; + SafeHandler::Ptr options; }; } @@ -94,6 +133,8 @@ class ConnectionPoolTest : public testing::Test { protected: ConnectionPoolTest() { + options = std::make_shared>(); + startServer(); } @@ -102,7 +143,7 @@ protected: DB::HTTPConnectionPools::Limits def_limits{}; DB::HTTPConnectionPools::instance().setLimits(def_limits, def_limits, def_limits); - setSlowDown(0); + options->set(RequestOptions()); DB::HTTPConnectionPools::instance().dropCache(); DB::CurrentThread::getProfileEvents().reset(); @@ -129,7 +170,7 @@ protected: void startServer() { server_data.reset(); - server_data.handler_factory = new HTTPRequestHandlerFactory(slowdown_receive); + server_data.handler_factory = new HTTPRequestHandlerFactory(options); server_data.server = std::make_unique( server_data.handler_factory, server_data.port); @@ -143,11 +184,20 @@ protected: void setSlowDown(size_t seconds) { - *slowdown_receive = seconds; + auto opt = options->get(); + opt.slowdown_receive = seconds; + options->set(std::move(opt)); + } + + void setOverWriteTimeout(size_t seconds) + { + auto opt = options->get(); + opt.overwrite_keep_alive_timeout = int(seconds); + options->set(std::move(opt)); } DB::ConnectionTimeouts timeouts; - std::shared_ptr> slowdown_receive = std::make_shared>(0); + SafeHandler::Ptr options; struct ServerData { @@ -182,7 +232,7 @@ protected: void wait_until(std::function pred) { while (!pred()) - sleepForMilliseconds(250); + sleepForMilliseconds(10); } void echoRequest(String data, HTTPSession & session) @@ -245,45 +295,52 @@ TEST_F(ConnectionPoolTest, CanRequest) ASSERT_EQ(0, getServer().currentConnections()); ASSERT_EQ(1, getServer().totalConnections()); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); + auto metrics = pool->getMetrics(); + + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + + ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); } TEST_F(ConnectionPoolTest, CanPreserve) { auto pool = getPool(); + auto metrics = pool->getMetrics(); { auto connection = pool->getConnection(timeouts); } - ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().active_count)); - ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().stored_count)); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + + ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count)); wait_until([&] () { return getServer().currentConnections() == 1; }); ASSERT_EQ(1, getServer().currentConnections()); - - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); } TEST_F(ConnectionPoolTest, CanReuse) { auto pool = getPool(); + auto metrics = pool->getMetrics(); { auto connection = pool->getConnection(timeouts); - // DB::setReuseTag(*connection); } - ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().active_count)); - ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().stored_count)); - { auto connection = pool->getConnection(timeouts); - ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().active_count)); - ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count)); + ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); wait_until([&] () { return getServer().currentConnections() == 1; }); ASSERT_EQ(1, getServer().currentConnections()); @@ -293,6 +350,11 @@ TEST_F(ConnectionPoolTest, CanReuse) ASSERT_EQ(1, getServer().totalConnections()); ASSERT_EQ(1, getServer().currentConnections()); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + connection->reset(); } @@ -303,15 +365,16 @@ TEST_F(ConnectionPoolTest, CanReuse) ASSERT_EQ(0, getServer().currentConnections()); ASSERT_EQ(1, getServer().totalConnections()); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); } TEST_F(ConnectionPoolTest, CanReuse10) { auto pool = getPool(); - + auto metrics = pool->getMetrics(); for (int i = 0; i < 10; ++i) { @@ -328,16 +391,23 @@ TEST_F(ConnectionPoolTest, CanReuse10) ASSERT_EQ(0, getServer().currentConnections()); ASSERT_EQ(1, getServer().totalConnections()); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); + + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); + + ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); } TEST_F(ConnectionPoolTest, CanReuse5) { - timeouts.withHTTPKeepAliveTimeout(1); + auto ka = Poco::Timespan(1, 0); // 1 seconds + timeouts.withHTTPKeepAliveTimeout(ka); auto pool = getPool(); + auto metrics = pool->getMetrics(); std::vector connections; connections.reserve(5); @@ -347,11 +417,14 @@ TEST_F(ConnectionPoolTest, CanReuse5) } connections.clear(); - ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); - ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().active_count)); - ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().stored_count)); + ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(5, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(5, CurrentMetrics::get(metrics.stored_count)); wait_until([&] () { return getServer().currentConnections() == 5; }); ASSERT_EQ(5, getServer().currentConnections()); @@ -363,35 +436,56 @@ TEST_F(ConnectionPoolTest, CanReuse5) echoRequest("Hello", *connection); } - ASSERT_EQ(5, getServer().totalConnections()); + ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); - ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); - ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().active_count)); - ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().stored_count)); + ASSERT_EQ(5, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(5, CurrentMetrics::get(metrics.stored_count)); + + /// wait until all connections are timeouted + wait_until([&] () { return getServer().currentConnections() == 0; }); + + { + // just to trigger pool->wipeExpired(); + auto connection = pool->getConnection(timeouts); + connection->reset(); + } + + ASSERT_EQ(6, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); } TEST_F(ConnectionPoolTest, CanReconnectAndCreate) { auto pool = getPool(); + auto metrics = pool->getMetrics(); std::vector in_use; - const size_t count = 2; + const size_t count = 3; for (int i = 0; i < count; ++i) { auto connection = pool->getConnection(timeouts); - // DB::setReuseTag(*connection); in_use.push_back(connection); } - ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); + ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); - ASSERT_EQ(count, CurrentMetrics::get(pool->getMetrics().active_count)); - ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count)); + ASSERT_EQ(count, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); auto connection = std::move(in_use.back()); in_use.pop_back(); @@ -402,28 +496,39 @@ TEST_F(ConnectionPoolTest, CanReconnectAndCreate) echoRequest("Hello", *connection); - connection->reset(); + ASSERT_EQ(count+1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); - wait_until([&] () { return getServer().currentConnections() == 1; }); - ASSERT_EQ(1, getServer().currentConnections()); - ASSERT_EQ(count+1, getServer().totalConnections()); - - ASSERT_EQ(count+1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); + ASSERT_EQ(count, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); } TEST_F(ConnectionPoolTest, CanReconnectAndReuse) { + auto ka = Poco::Timespan(1, 0); // 1 seconds + timeouts.withHTTPKeepAliveTimeout(ka); + auto pool = getPool(); + auto metrics = pool->getMetrics(); std::vector in_use; - const size_t count = 2; + const size_t count = 3; + for (int i = 0; i < count; ++i) + { + auto connection = pool->getConnection(timeouts); + /// make some request in order to show to the server the keep alive headers + echoRequest("Hello", *connection); + in_use.push_back(std::move(connection)); + } + in_use.clear(); + for (int i = 0; i < count; ++i) { auto connection = pool->getConnection(timeouts); - // DB::setReuseTag(*connection); in_use.push_back(std::move(connection)); } @@ -441,11 +546,16 @@ TEST_F(ConnectionPoolTest, CanReconnectAndReuse) wait_until([&] () { return getServer().currentConnections() == 0; }); ASSERT_EQ(0, getServer().currentConnections()); - ASSERT_EQ(2, getServer().totalConnections()); + ASSERT_EQ(count, getServer().totalConnections()); - ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); + ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(count + count - 1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(count + 1, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(count-1, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(count-2, CurrentMetrics::get(metrics.stored_count)); } TEST_F(ConnectionPoolTest, ReceiveTimeout) @@ -454,6 +564,7 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout) timeouts.withReceiveTimeout(1); auto pool = getPool(); + auto metrics = pool->getMetrics(); { auto connection = pool->getConnection(timeouts); @@ -462,10 +573,14 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout) ); } - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); { timeouts.withReceiveTimeout(3); @@ -475,10 +590,14 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout) ); } - ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]); + ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count)); { /// timeouts have effect for reused session @@ -489,10 +608,14 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout) ); } - ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]); - ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]); + ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); } TEST_F(ConnectionPoolTest, ReadWriteBufferFromHTTP) @@ -500,6 +623,7 @@ TEST_F(ConnectionPoolTest, ReadWriteBufferFromHTTP) std::string_view message = "Hello ReadWriteBufferFromHTTP"; auto uri = Poco::URI(getServerUrl()); auto metrics = DB::HTTPConnectionPools::instance().getPool(DB::HTTPConnectionGroupType::HTTP, uri, DB::ProxyConfiguration{})->getMetrics(); + Poco::Net::HTTPBasicCredentials empty_creds; auto buf_from_http = DB::BuilderRWBufferFromHTTP(uri) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) @@ -527,6 +651,7 @@ TEST_F(ConnectionPoolTest, ReadWriteBufferFromHTTP) ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count)); @@ -538,23 +663,26 @@ TEST_F(ConnectionPoolTest, HardLimit) DB::HTTPConnectionPools::instance().setLimits(zero_limits, zero_limits, zero_limits); auto pool = getPool(); + auto metrics = pool->getMetrics(); { auto connection = pool->getConnection(timeouts); } - ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().active_count)); - ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count)); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); - - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]); + ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); } TEST_F(ConnectionPoolTest, NoReceiveCall) { auto pool = getPool(); + auto metrics = pool->getMetrics(); { auto connection = pool->getConnection(timeouts); @@ -570,11 +698,124 @@ TEST_F(ConnectionPoolTest, NoReceiveCall) connection->flushRequest(); } - ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().active_count)); - ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count)); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); - - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]); - ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]); - ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]); + ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); +} + +TEST_F(ConnectionPoolTest, ReconnectedWhenConnectionIsHoldTooLong) +{ + auto ka = Poco::Timespan(1, 0); // 1 seconds + timeouts.withHTTPKeepAliveTimeout(ka); + + auto pool = getPool(); + auto metrics = pool->getMetrics(); + + { + auto connection = pool->getConnection(timeouts); + + echoRequest("Hello", *connection); + + auto fake_ka = Poco::Timespan(30 * 1000 * 1000); // 30 seconds + timeouts.withHTTPKeepAliveTimeout(fake_ka); + DB::setTimeouts(*connection, timeouts); // new keep alive timeout has no effect + + wait_until([&] () { return getServer().currentConnections() == 0; }); + + ASSERT_EQ(1, connection->connected()); + ASSERT_EQ(1, connection->getKeepAlive()); + ASSERT_EQ(1000, connection->getKeepAliveTimeout().totalMilliseconds()); + ASSERT_EQ(1, connection->isKeepAliveExpired(connection->getKeepAliveReliability())); + + echoRequest("Hello", *connection); + } + + + ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count)); +} + +TEST_F(ConnectionPoolTest, ReconnectedWhenConnectionIsNearlyExpired) +{ + auto ka = Poco::Timespan(1, 0); // 1 seconds + timeouts.withHTTPKeepAliveTimeout(ka); + + auto pool = getPool(); + auto metrics = pool->getMetrics(); + + { + { + auto connection = pool->getConnection(timeouts); + echoRequest("Hello", *connection); + } + + sleepForMilliseconds(900); + + { + auto connection = pool->getConnection(timeouts); + echoRequest("Hello", *connection); + } + } + + ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count)); +} + +TEST_F(ConnectionPoolTest, ServerOverwriteKeepAlive) +{ + auto ka = Poco::Timespan(30, 0); // 30 seconds + timeouts.withHTTPKeepAliveTimeout(ka); + + auto pool = getPool(); + auto metrics = pool->getMetrics(); + + { + auto connection = pool->getConnection(timeouts); + echoRequest("Hello", *connection); + ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds()); + ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds()); + } + + { + setOverWriteTimeout(1); + auto connection = pool->getConnection(timeouts); + echoRequest("Hello", *connection); + ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds()); + ASSERT_EQ(1, connection->getKeepAliveTimeout().totalSeconds()); + } + + { + // server do not overwrite it in the following requests but client has to remember last agreed value + setOverWriteTimeout(0); + auto connection = pool->getConnection(timeouts); + echoRequest("Hello", *connection); + ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds()); + ASSERT_EQ(1, connection->getKeepAliveTimeout().totalSeconds()); + } + + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(3, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count)); } diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 6608a35a5a2..8d6d8ebc1a2 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -128,9 +128,9 @@ namespace DB M(Bool, format_alter_operations_with_parentheses, false, "If enabled, each operation in alter queries will be surrounded with parentheses in formatted queries to make them less ambiguous.", 0) \ M(String, default_replica_path, "/clickhouse/tables/{uuid}/{shard}", "The path to the table in ZooKeeper", 0) \ M(String, default_replica_name, "{replica}", "The replica name in ZooKeeper", 0) \ - M(UInt64, disk_connections_soft_limit, 1000, "Connections above this limit have significantly shorter time to live. The limit applies to the disks connections.", 0) \ + M(UInt64, disk_connections_soft_limit, 5000, "Connections above this limit have significantly shorter time to live. The limit applies to the disks connections.", 0) \ M(UInt64, disk_connections_warn_limit, 10000, "Warning massages are written to the logs if number of in-use connections are higher than this limit. The limit applies to the disks connections.", 0) \ - M(UInt64, disk_connections_store_limit, 12000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the disks connections.", 0) \ + M(UInt64, disk_connections_store_limit, 30000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the disks connections.", 0) \ M(UInt64, storage_connections_soft_limit, 100, "Connections above this limit have significantly shorter time to live. The limit applies to the storages connections.", 0) \ M(UInt64, storage_connections_warn_limit, 1000, "Warning massages are written to the logs if number of in-use connections are higher than this limit. The limit applies to the storages connections.", 0) \ M(UInt64, storage_connections_store_limit, 5000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the storages connections.", 0) \ diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index c4b636103fe..8813c958185 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -144,7 +144,11 @@ ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(const String & method void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) { session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); - session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); + /// we can not change keep alive timeout for already initiated connections + if (!session.connected()) + { + session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); + } } ConnectionTimeouts getTimeouts(const Poco::Net::HTTPClientSession & session) From 0bc743b4e2144d6de39500aa753d9d02675caa18 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 15:24:16 +0200 Subject: [PATCH 0937/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/35608 --- ...alyzer_numeric_literals_as_column_names.reference | 2 ++ ...079_analyzer_numeric_literals_as_column_names.sql | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.reference create mode 100644 tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql diff --git a/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.reference b/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.reference new file mode 100644 index 00000000000..faff07c519f --- /dev/null +++ b/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.reference @@ -0,0 +1,2 @@ +\N 1 +str diff --git a/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql b/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql new file mode 100644 index 00000000000..e6bcad34506 --- /dev/null +++ b/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql @@ -0,0 +1,12 @@ +CREATE TABLE testdata (`1` String) ENGINE=MergeTree ORDER BY tuple(); +INSERT INTO testdata VALUES ('testdata'); + +SELECT * +FROM ( + SELECT if(isValidUTF8(`1`), NULL, 'error!') AS error_message, + if(error_message IS NULL, 1, 0) AS valid + FROM testdata +) +WHERE valid; + +select * from (select 'str' as `1`) where 1; From 6c14f6ecf2aba16e407fe887ef38e59d9931510d Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 3 Apr 2024 13:25:04 +0000 Subject: [PATCH 0938/1165] Close: https://github.com/ClickHouse/ClickHouse/pull/62185 --- .../03033_cte_numbers_memory.reference | 10 ++++++++++ .../0_stateless/03033_cte_numbers_memory.sql | 15 +++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/queries/0_stateless/03033_cte_numbers_memory.reference create mode 100644 tests/queries/0_stateless/03033_cte_numbers_memory.sql diff --git a/tests/queries/0_stateless/03033_cte_numbers_memory.reference b/tests/queries/0_stateless/03033_cte_numbers_memory.reference new file mode 100644 index 00000000000..8d2470dea44 --- /dev/null +++ b/tests/queries/0_stateless/03033_cte_numbers_memory.reference @@ -0,0 +1,10 @@ +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 diff --git a/tests/queries/0_stateless/03033_cte_numbers_memory.sql b/tests/queries/0_stateless/03033_cte_numbers_memory.sql new file mode 100644 index 00000000000..0e3ee9abd65 --- /dev/null +++ b/tests/queries/0_stateless/03033_cte_numbers_memory.sql @@ -0,0 +1,15 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/61238 + +WITH +(SELECT number FROM system.numbers LIMIT 1) as w1, +(SELECT number FROM system.numbers LIMIT 1) as w2, +(SELECT number FROM system.numbers LIMIT 1) as w3, +(SELECT number FROM system.numbers LIMIT 1) as w4, +(SELECT number FROM system.numbers LIMIT 1) as w5, +(SELECT number FROM system.numbers LIMIT 1) as w6 +SELECT number FROM ( + SELECT number FROM system.numbers LIMIT 10 + UNION ALL + SELECT number FROM system.numbers LIMIT 10 +) +WHERE number < 5; From 922a14eaf1fd22d1a364ec285851c50cbb2ad54f Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 3 Apr 2024 15:33:35 +0200 Subject: [PATCH 0939/1165] fix stored_count metric --- src/Common/HTTPConnectionPool.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index 21165bbc62d..ab0ea1571d3 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -560,6 +560,11 @@ public: size_t wipeExpiredImpl(std::vector & expired_connections) TSA_REQUIRES(mutex) { + SCOPE_EXIT({ + CurrentMetrics::sub(getMetrics().stored_count, expired_connections.size()); + ProfileEvents::increment(getMetrics().expired, expired_connections.size()); + }); + auto isSoftLimitReached = group->isSoftLimitReached(); while (!stored_connections.empty()) { @@ -573,9 +578,6 @@ public: expired_connections.push_back(connection); } - CurrentMetrics::sub(getMetrics().stored_count, expired_connections.size()); - ProfileEvents::increment(getMetrics().expired, expired_connections.size()); - return stored_connections.size(); } From 394a4f39ecf12867344f68ac4730ea34abf8b0dc Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 3 Apr 2024 10:37:39 -0300 Subject: [PATCH 0940/1165] remove unnecessary matching character --- src/IO/S3/URI.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 7f628d27f34..8e364337db5 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -40,7 +40,7 @@ URI::URI(const std::string & uri_) /// Case when AWS Private Link Interface is being used /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html - static const RE2 aws_private_link_style_pattern(R"(bucket\.vpce\-([a-z0-9\-.:]+)\.vpce.amazonaws.com)"); + static const RE2 aws_private_link_style_pattern(R"(bucket\.vpce\-([a-z0-9\-.]+)\.vpce.amazonaws.com)"); /// Case when bucket name and key represented in path of S3 URL. /// E.g. (https://s3.region.amazonaws.com/bucket-name/key) From ee1c177d10a3c333662ba774af7fb2618193fac1 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 3 Apr 2024 13:49:53 +0000 Subject: [PATCH 0941/1165] Closes: https://github.com/ClickHouse/ClickHouse/issues/29838 --- .../03080_incorrect_join_with_merge.reference | 2 + .../03080_incorrect_join_with_merge.sql | 66 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 tests/queries/0_stateless/03080_incorrect_join_with_merge.reference create mode 100644 tests/queries/0_stateless/03080_incorrect_join_with_merge.sql diff --git a/tests/queries/0_stateless/03080_incorrect_join_with_merge.reference b/tests/queries/0_stateless/03080_incorrect_join_with_merge.reference new file mode 100644 index 00000000000..51993f072d5 --- /dev/null +++ b/tests/queries/0_stateless/03080_incorrect_join_with_merge.reference @@ -0,0 +1,2 @@ +2 +2 diff --git a/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql b/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql new file mode 100644 index 00000000000..ae8e40f6d56 --- /dev/null +++ b/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql @@ -0,0 +1,66 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/29838 + +CREATE TABLE first_table_lr +( + id String, + id2 String +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_03080/alter', 'r1') +ORDER BY id; + + +CREATE TABLE first_table +( + id String, + id2 String +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), 'first_table_lr'); + + +CREATE TABLE second_table_lr +( + id String, + id2 String +) ENGINE = MergeTree() +ORDER BY id; + +CREATE TABLE second_table +( + id String, + id2 String +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), 'second_table_lr'); + +INSERT INTO first_table VALUES ('1', '2'), ('3', '4'); +INSERT INTO second_table VALUES ('1', '2'), ('3', '4'); + +CREATE TABLE two_tables +( + id String, + id2 String +) +ENGINE = Merge(currentDatabase(), '^(first_table)$'); + +SELECT + count() +FROM first_table as s +GLOBAL ANY JOIN second_table as f USING (id) +WHERE + f.id2 GLOBAL IN ( + SELECT + id2 + FROM second_table + GROUP BY id2 + ); + +SELECT + count() +FROM two_tables as s +GLOBAL ANY JOIN second_table as f USING (id) +WHERE + f.id2 GLOBAL IN ( + SELECT + id2 + FROM second_table + GROUP BY id2 + ); From 4bb8f12374dc58fc87ed76fb7abe04b926500ec8 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 15:32:12 +0200 Subject: [PATCH 0942/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/35652 --- ...n_name_to_alias__virtual_columns.reference | 3 +++ ..._column_name_to_alias__virtual_columns.sql | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.reference create mode 100644 tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql diff --git a/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.reference b/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.reference new file mode 100644 index 00000000000..ccf161abe8d --- /dev/null +++ b/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.reference @@ -0,0 +1,3 @@ +0 0 +0 0 + 0 0 diff --git a/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql b/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql new file mode 100644 index 00000000000..1fe19cdad2a --- /dev/null +++ b/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql @@ -0,0 +1,27 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/35652 +CREATE TABLE test ( + id UInt64 +) +ENGINE = MergeTree() +SAMPLE BY intHash32(id) +ORDER BY intHash32(id); + +SELECT + any(id), + any(id) AS id +FROM test +SETTINGS prefer_column_name_to_alias = 1; + +SELECT + any(_sample_factor), + any(_sample_factor) AS _sample_factor +FROM test +SETTINGS prefer_column_name_to_alias = 1; + +SELECT + any(_partition_id), + any(_sample_factor), + any(_partition_id) AS _partition_id, + any(_sample_factor) AS _sample_factor +FROM test +SETTINGS prefer_column_name_to_alias = 1; From c539a6ba8d4fc5b6adc8e2ef9a240db37dda3550 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 15:41:40 +0200 Subject: [PATCH 0943/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/36189 --- .../03081_analyzer_agg_func_CTE.reference | 2 ++ .../03081_analyzer_agg_func_CTE.sql | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/queries/0_stateless/03081_analyzer_agg_func_CTE.reference create mode 100644 tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql diff --git a/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.reference b/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.reference new file mode 100644 index 00000000000..bf1b8926236 --- /dev/null +++ b/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.reference @@ -0,0 +1,2 @@ +2020-01-01 +9 diff --git a/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql b/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql new file mode 100644 index 00000000000..81dbbb3b62d --- /dev/null +++ b/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql @@ -0,0 +1,18 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/36189 +CREATE TABLE test +( + `dt` Date, + `text` String +) +ENGINE = MergeTree +ORDER BY dt; + +insert into test values ('2020-01-01', 'text1'), ('2019-01-01', 'text2'), ('1900-01-01', 'text3'); + +WITH max(dt) AS maxDt +SELECT maxDt +FROM test; + +WITH max(number) AS maxDt +SELECT maxDt +FROM numbers(10); From 0cda6cf5233a1d24e3c02ea402ee3be4bfa61625 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 15:51:16 +0200 Subject: [PATCH 0944/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/39634 --- ...nalyzer_left_join_correct_column.reference | 2 ++ ...3082_analyzer_left_join_correct_column.sql | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/queries/0_stateless/03082_analyzer_left_join_correct_column.reference create mode 100644 tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql diff --git a/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.reference b/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.reference new file mode 100644 index 00000000000..ad3f3d53ab5 --- /dev/null +++ b/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.reference @@ -0,0 +1,2 @@ +pk1 2 +pk1 2 diff --git a/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql b/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql new file mode 100644 index 00000000000..93702fee550 --- /dev/null +++ b/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql @@ -0,0 +1,30 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/39634 +CREATE TABLE test1 +( + `pk` String, + `x.y` Decimal(18, 4) +) +ENGINE = MergeTree() +ORDER BY (pk); + +CREATE TABLE test2 +( + `pk` String, + `x.y` Decimal(18, 4) +) +ENGINE = MergeTree() +ORDER BY (pk); + +INSERT INTO test1 SELECT 'pk1', 1; + +INSERT INTO test2 SELECT 'pk1', 2; + +SELECT t1.pk, t2.x.y +FROM test1 t1 +LEFT JOIN test2 t2 + on t1.pk = t2.pk; + +SELECT t1.pk, t2.`x.y` +FROM test1 t1 +LEFT JOIN test2 t2 + on t1.pk = t2.pk; From 532d80e20b60987947ac11eb8c4991916742157f Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 3 Apr 2024 16:02:07 +0200 Subject: [PATCH 0945/1165] fix log level in debug code --- src/Common/HTTPConnectionPool.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index ab0ea1571d3..2d3a87dda6b 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -361,13 +361,16 @@ private: std::istream & receiveResponse(Poco::Net::HTTPResponse & response) override { + int originKA = Session::getKeepAliveTimeout().totalSeconds(); + std::istream & result = Session::receiveResponse(response); result.exceptions(std::ios::badbit); // that line is for temporary debug, will be removed if (response.has(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE)) - LOG_WARNING(log, "received keep alive header: {}", - response.get(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE, Poco::Net::HTTPMessage::EMPTY)); + LOG_INFO(log, "received keep alive header: {}, original was {}", + response.get(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE, Poco::Net::HTTPMessage::EMPTY), + originKA); response_stream = &result; response_stream_completed = false; From d19d09e31e55e60bbf97a8636946e80d4e3f6688 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 3 Apr 2024 16:17:56 +0200 Subject: [PATCH 0946/1165] Correctly handle const columns in DistinctTransfom --- src/Processors/Transforms/DistinctTransform.cpp | 1 + .../0_stateless/03033_distinct_transform_const_columns.reference | 1 + .../0_stateless/03033_distinct_transform_const_columns.sql | 1 + 3 files changed, 3 insertions(+) create mode 100644 tests/queries/0_stateless/03033_distinct_transform_const_columns.reference create mode 100644 tests/queries/0_stateless/03033_distinct_transform_const_columns.sql diff --git a/src/Processors/Transforms/DistinctTransform.cpp b/src/Processors/Transforms/DistinctTransform.cpp index 3619fa51bf6..d528303a642 100644 --- a/src/Processors/Transforms/DistinctTransform.cpp +++ b/src/Processors/Transforms/DistinctTransform.cpp @@ -55,6 +55,7 @@ void DistinctTransform::transform(Chunk & chunk) /// Convert to full column, because SetVariant for sparse column is not implemented. convertToFullIfSparse(chunk); + convertToFullIfConst(chunk); const auto num_rows = chunk.getNumRows(); auto columns = chunk.detachColumns(); diff --git a/tests/queries/0_stateless/03033_distinct_transform_const_columns.reference b/tests/queries/0_stateless/03033_distinct_transform_const_columns.reference new file mode 100644 index 00000000000..d05b1f927f4 --- /dev/null +++ b/tests/queries/0_stateless/03033_distinct_transform_const_columns.reference @@ -0,0 +1 @@ +0 0 diff --git a/tests/queries/0_stateless/03033_distinct_transform_const_columns.sql b/tests/queries/0_stateless/03033_distinct_transform_const_columns.sql new file mode 100644 index 00000000000..41df19ab64e --- /dev/null +++ b/tests/queries/0_stateless/03033_distinct_transform_const_columns.sql @@ -0,0 +1 @@ +SELECT DISTINCT COALESCE(COALESCE('') = toNullable('b3'), toUInt128(toNullable(2)), 2, 2, toLowCardinality(2), 2, 2, 2, toUInt128(toNullable(2)), materialize(2), toUInt128(2), 2, 2), COALESCE(COALESCE(COALESCE(materialize(''))) = 'b3', 2, 2, 2, toLowCardinality(2), toUInt128(2), 2, 2, 2, materialize(toUInt256(2)), 2, 2, 2) FROM numbers(100000); From 6661484e555081a01a587ffe1b0174baad11a7af Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 3 Apr 2024 16:31:01 +0200 Subject: [PATCH 0947/1165] Proper fix for LowCardinality together with JSONExtact functions (#61957) --- src/Functions/FunctionsJSON.h | 71 ++++++++++++++++--- .../00918_json_functions.reference | 6 ++ .../0_stateless/00918_json_functions.sql | 6 ++ ...74_extract_fixedstring_from_json.reference | 7 ++ .../02474_extract_fixedstring_from_json.sql | 7 ++ 5 files changed, 89 insertions(+), 8 deletions(-) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 2539fa1aeb4..9e824fabc42 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -257,7 +257,7 @@ private: } case MoveType::Key: { - key = (*arguments[j + 1].column).getDataAt(row).toView(); + key = arguments[j + 1].column->getDataAt(row).toView(); if (!moveToElementByKey(res_element, key)) return false; break; @@ -334,6 +334,26 @@ private: }; +template +class JSONExtractImpl; + +template +class JSONExtractKeysAndValuesImpl; + +/** +* Functions JSONExtract and JSONExtractKeysAndValues force the return type - it is specified in the last argument. +* For example - `SELECT JSONExtract(materialize('{"a": 131231, "b": 1234}'), 'b', 'LowCardinality(FixedString(4))')` +* But by default ClickHouse decides on its own whether the return type will be LowCardinality based on the types of +* input arguments. +* And for these specific functions we cannot rely on this mechanism, so these functions have their own implementation - +* just convert all of the LowCardinality input columns to full ones, execute and wrap the resulting column in LowCardinality +* if needed. +*/ +template typename Impl> +constexpr bool functionForcesTheReturnType() +{ + return std::is_same_v, JSONExtractImpl> || std::is_same_v, JSONExtractKeysAndValuesImpl>; +} template typename Impl> class ExecutableFunctionJSON : public IExecutableFunction @@ -348,17 +368,50 @@ public: String getName() const override { return Name::name; } bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } + bool useDefaultImplementationForLowCardinalityColumns() const override + { + return !functionForcesTheReturnType(); + } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (null_presence.has_null_constant) return result_type->createColumnConstWithDefaultValue(input_rows_count); - ColumnsWithTypeAndName temporary_columns = null_presence.has_nullable ? createBlockWithNestedColumns(arguments) : arguments; - ColumnPtr temporary_result = chooseAndRunJSONParser(temporary_columns, json_return_type, input_rows_count); - if (null_presence.has_nullable) - return wrapInNullable(temporary_result, arguments, result_type, input_rows_count); - return temporary_result; + if constexpr (functionForcesTheReturnType()) + { + ColumnsWithTypeAndName columns_without_low_cardinality = arguments; + + for (auto & column : columns_without_low_cardinality) + { + column.column = recursiveRemoveLowCardinality(column.column); + column.type = recursiveRemoveLowCardinality(column.type); + } + + ColumnsWithTypeAndName temporary_columns = null_presence.has_nullable ? createBlockWithNestedColumns(columns_without_low_cardinality) : columns_without_low_cardinality; + ColumnPtr temporary_result = chooseAndRunJSONParser(temporary_columns, json_return_type, input_rows_count); + + if (null_presence.has_nullable) + temporary_result = wrapInNullable(temporary_result, columns_without_low_cardinality, result_type, input_rows_count); + + if (result_type->lowCardinality()) + temporary_result = recursiveLowCardinalityTypeConversion(temporary_result, json_return_type, result_type); + + return temporary_result; + } + else + { + ColumnsWithTypeAndName temporary_columns = null_presence.has_nullable ? createBlockWithNestedColumns(arguments) : arguments; + ColumnPtr temporary_result = chooseAndRunJSONParser(temporary_columns, json_return_type, input_rows_count); + + if (null_presence.has_nullable) + temporary_result = wrapInNullable(temporary_result, arguments, result_type, input_rows_count); + + if (result_type->lowCardinality()) + temporary_result = recursiveLowCardinalityTypeConversion(temporary_result, json_return_type, result_type); + + return temporary_result; + } } private: @@ -429,7 +482,6 @@ private: DataTypePtr json_return_type; }; - /// We use IFunctionOverloadResolver instead of IFunction to handle non-default NULL processing. /// Both NULL and JSON NULL should generate NULL value. If any argument is NULL, return NULL. template typename Impl> @@ -450,6 +502,10 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override + { + return !functionForcesTheReturnType(); + } FunctionBasePtr build(const ColumnsWithTypeAndName & arguments) const override { @@ -481,7 +537,6 @@ public: } }; - struct NameJSONHas { static constexpr auto name{"JSONHas"}; }; struct NameIsValidJSON { static constexpr auto name{"isValidJSON"}; }; struct NameJSONLength { static constexpr auto name{"JSONLength"}; }; diff --git a/tests/queries/0_stateless/00918_json_functions.reference b/tests/queries/0_stateless/00918_json_functions.reference index 43b15ded93d..078348cd20f 100644 --- a/tests/queries/0_stateless/00918_json_functions.reference +++ b/tests/queries/0_stateless/00918_json_functions.reference @@ -286,3 +286,9 @@ v --show error: type should be const string --show error: index type should be integer --show error: key of map type should be String +\N +\N +Hello +Hello +Hello +Hello diff --git a/tests/queries/0_stateless/00918_json_functions.sql b/tests/queries/0_stateless/00918_json_functions.sql index e19dd17670e..3d30ce841ba 100644 --- a/tests/queries/0_stateless/00918_json_functions.sql +++ b/tests/queries/0_stateless/00918_json_functions.sql @@ -326,3 +326,9 @@ SELECT JSONExtract('[]', JSONExtract('0', 'UInt256'), 'UInt256'); -- { serverErr SELECT '--show error: key of map type should be String'; SELECT JSONExtract('{"a": [100.0, 200], "b": [-100, 200.0, 300]}', 'Map(Int64, Array(Float64))'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT JSONExtract(materialize(toLowCardinality('{"string_value":null}')), materialize('string_value'), 'LowCardinality(Nullable(String))'); +SELECT JSONExtract(materialize('{"string_value":null}'), materialize('string_value'), 'LowCardinality(Nullable(String))'); +SELECT JSONExtract(materialize('{"string_value":"Hello"}'), materialize('string_value'), 'LowCardinality(Nullable(String))') AS x; +SELECT JSONExtract(materialize(toLowCardinality('{"string_value":"Hello"}')), materialize('string_value'), 'LowCardinality(Nullable(String))') AS x; +SELECT JSONExtract(materialize('{"string_value":"Hello"}'), materialize(toLowCardinality('string_value')), 'LowCardinality(Nullable(String))') AS x; +SELECT JSONExtract(materialize(toLowCardinality('{"string_value":"Hello"}')), materialize(toLowCardinality('string_value')), 'LowCardinality(Nullable(String))') AS x; diff --git a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference index 783d12fcf1a..21ddf5d3512 100644 --- a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference +++ b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.reference @@ -8,3 +8,10 @@ \0\0\0\0\0 131231 131231 +1234 +1234 +{"b":131231} +\0\0\0\0 +1234567890 +18446744073709551615 +-9223372036854775807 diff --git a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql index cfc47e00cba..bbb9f55062b 100644 --- a/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql +++ b/tests/queries/0_stateless/02474_extract_fixedstring_from_json.sql @@ -6,3 +6,10 @@ SELECT JSONExtract('{"a": 123456}', 'a', 'FixedString(5)'); SELECT JSONExtract('{"a": 123456}', 'a', 'FixedString(6)'); SELECT JSONExtract(materialize('{"a": 131231}'), 'a', 'LowCardinality(FixedString(5))') FROM numbers(2); SELECT JSONExtract(materialize('{"a": 131231}'), 'a', 'LowCardinality(FixedString(6))') FROM numbers(2); +SELECT JSONExtract(materialize('{"a": 131231, "b": 1234}'), 'b', 'LowCardinality(FixedString(4))'); +SELECT JSONExtract(materialize('{"a": 131231, "b": "1234"}'), 'b', 'LowCardinality(FixedString(4))'); +SELECT JSONExtract(materialize('{"a": {"b": 131231} }'), 'a', 'LowCardinality(FixedString(12))'); +SELECT JSONExtract(materialize('{"a": 131231, "b": 1234567890}'), 'b', 'LowCardinality(FixedString(4))'); +SELECT JSONExtract(materialize('{"a": 131231, "b": 1234567890}'), 'b', 'LowCardinality(FixedString(10))'); +SELECT JSONExtract(materialize('{"a": 18446744073709551615}'), 'a', 'LowCardinality(FixedString(20))'); +SELECT JSONExtract(materialize('{"a": -9223372036854775807}'), 'a', 'LowCardinality(FixedString(20))'); From 0b53980221d419fc1d0227c851502cdaeafc7892 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 16:45:56 +0200 Subject: [PATCH 0948/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/47432 --- ...03084_analyzer_join_column_alias.reference | 1 + .../03084_analyzer_join_column_alias.sql | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/queries/0_stateless/03084_analyzer_join_column_alias.reference create mode 100644 tests/queries/0_stateless/03084_analyzer_join_column_alias.sql diff --git a/tests/queries/0_stateless/03084_analyzer_join_column_alias.reference b/tests/queries/0_stateless/03084_analyzer_join_column_alias.reference new file mode 100644 index 00000000000..acbb8406347 --- /dev/null +++ b/tests/queries/0_stateless/03084_analyzer_join_column_alias.reference @@ -0,0 +1 @@ +2023-01-01 diff --git a/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql b/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql new file mode 100644 index 00000000000..caf65823532 --- /dev/null +++ b/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql @@ -0,0 +1,23 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/47432 +create or replace table t1 +engine = MergeTree() +order by tuple() +as +select 1 as user_id, 2 as level; + + +create or replace table t2 +engine = MergeTree() +order by tuple() +as +select 1 as user_id, 'website' as event_source, '2023-01-01 00:00:00'::DateTime as timestamp; + + +alter table t2 +add column date Date alias toDate(timestamp); + +SELECT + any(t2.date) as any_val +FROM t1 AS t1 +LEFT JOIN t2 as t2 + ON (t1.user_id = t2.user_id); From bc9d380e5a88be17cda135c9f649103404204bb2 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 16:46:45 +0200 Subject: [PATCH 0949/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/54910 --- .../03085_analyzer_alias_column_group_by.reference | 1 + .../0_stateless/03085_analyzer_alias_column_group_by.sql | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 tests/queries/0_stateless/03085_analyzer_alias_column_group_by.reference create mode 100644 tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql diff --git a/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.reference b/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.reference new file mode 100644 index 00000000000..804f12662b8 --- /dev/null +++ b/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.reference @@ -0,0 +1 @@ +String 1 diff --git a/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql b/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql new file mode 100644 index 00000000000..f4eaa5d9710 --- /dev/null +++ b/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql @@ -0,0 +1,4 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/54910 +SELECT toTypeName(stat_standard_id) AS stat_standard_id_1, count(1) AS value +FROM ( SELECT 'string value' AS stat_standard_id ) +GROUP BY stat_standard_id_1 LIMIT 1 From a5107417449477f1dddb0fd76502c56652a4dd4a Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 16:49:35 +0200 Subject: [PATCH 0950/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/57321 --- ...86_analyzer_window_func_part_of_group_by.reference | 2 ++ .../03086_analyzer_window_func_part_of_group_by.sql | 11 +++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.reference create mode 100644 tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql diff --git a/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.reference b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.reference new file mode 100644 index 00000000000..c8b2d51ae53 --- /dev/null +++ b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.reference @@ -0,0 +1,2 @@ +1 2 +2 2 diff --git a/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql new file mode 100644 index 00000000000..55a60873a5a --- /dev/null +++ b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql @@ -0,0 +1,11 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/57321 +SELECT + ver, + max(ver) OVER () AS ver_max +FROM +( + SELECT 1 AS ver + UNION ALL + SELECT 2 AS ver +) +GROUP BY ver From 346a0ec53b1dd2984ee4fe55785d61cec3a5b11b Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 17:05:48 +0200 Subject: [PATCH 0951/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/59154 --- .../03087_analyzer_subquery_with_alias.reference | 1 + .../03087_analyzer_subquery_with_alias.sql | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/queries/0_stateless/03087_analyzer_subquery_with_alias.reference create mode 100644 tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql diff --git a/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.reference b/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.reference new file mode 100644 index 00000000000..7660873d103 --- /dev/null +++ b/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.reference @@ -0,0 +1 @@ +[1] diff --git a/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql b/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql new file mode 100644 index 00000000000..98aca76fe49 --- /dev/null +++ b/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql @@ -0,0 +1,15 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/59154 +SELECT * +FROM +( + WITH + assumeNotNull(( + SELECT 0.9 + )) AS TUNING, + ELEMENT_QUERY AS + ( + SELECT quantiles(TUNING)(1) + ) + SELECT * + FROM ELEMENT_QUERY +); From bbcecd26606df86e4c2359c2bab811892dd8d4f1 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:07:44 +0200 Subject: [PATCH 0952/1165] Remove reverted PR from 24.3 changelog --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84e51c1efdf..dd88f3ee2c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,7 +123,6 @@ * Something was wrong with Apache Hive, which is experimental and not supported. [#60262](https://github.com/ClickHouse/ClickHouse/pull/60262) ([shanfengp](https://github.com/Aed-p)). * An improvement for experimental parallel replicas: force reanalysis if parallel replicas changed [#60362](https://github.com/ClickHouse/ClickHouse/pull/60362) ([Raúl Marín](https://github.com/Algunenano)). * Fix usage of plain metadata type with new disks configuration option [#60396](https://github.com/ClickHouse/ClickHouse/pull/60396) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Don't allow to set max_parallel_replicas to 0 as it doesn't make sense [#60430](https://github.com/ClickHouse/ClickHouse/pull/60430) ([Kruglov Pavel](https://github.com/Avogar)). * Try to fix logical error 'Cannot capture column because it has incompatible type' in mapContainsKeyLike [#60451](https://github.com/ClickHouse/ClickHouse/pull/60451) ([Kruglov Pavel](https://github.com/Avogar)). * Avoid calculation of scalar subqueries for CREATE TABLE. [#60464](https://github.com/ClickHouse/ClickHouse/pull/60464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix deadlock in parallel parsing when lots of rows are skipped due to errors [#60516](https://github.com/ClickHouse/ClickHouse/pull/60516) ([Kruglov Pavel](https://github.com/Avogar)). From 39205fe676b97923765a7fe8791f68a7f915de35 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 17:10:10 +0200 Subject: [PATCH 0953/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/61014 --- ...3088_analyzer_ambiguous_column_multi_call.reference | 0 .../03088_analyzer_ambiguous_column_multi_call.sql | 10 ++++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.reference create mode 100644 tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql diff --git a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.reference b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql new file mode 100644 index 00000000000..84afdb295c2 --- /dev/null +++ b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql @@ -0,0 +1,10 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/61014 +create database test_03088; + +create table test_03088.a (i int) engine = Log(); + +select + test_03088.a.i +from + test_03088.a, + test_03088.a as x; From ccf0953d8013284815102643cb735d6151352c04 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 3 Apr 2024 17:14:05 +0200 Subject: [PATCH 0954/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/61950 --- .../03089_analyzer_alias_replacement.reference | 2 ++ .../0_stateless/03089_analyzer_alias_replacement.sql | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03089_analyzer_alias_replacement.reference create mode 100644 tests/queries/0_stateless/03089_analyzer_alias_replacement.sql diff --git a/tests/queries/0_stateless/03089_analyzer_alias_replacement.reference b/tests/queries/0_stateless/03089_analyzer_alias_replacement.reference new file mode 100644 index 00000000000..2f1b638ff54 --- /dev/null +++ b/tests/queries/0_stateless/03089_analyzer_alias_replacement.reference @@ -0,0 +1,2 @@ +1 +4 diff --git a/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql b/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql new file mode 100644 index 00000000000..00a3795eab8 --- /dev/null +++ b/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql @@ -0,0 +1,8 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/61950 + +with dummy + 1 as dummy select dummy from system.one; + +WITH dummy + 3 AS dummy +SELECT dummy + 1 AS y +FROM system.one +SETTINGS enable_global_with_statement = 1; From 5fadac4994f316761a0427702459ab604b0d8c8b Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 3 Apr 2024 15:24:44 +0000 Subject: [PATCH 0955/1165] Update contrib to new commit --- contrib/arrow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/arrow b/contrib/arrow index 46e7ed11c2e..8f36d71d185 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 46e7ed11c2e0ef62ccbbe23e6a35a4988884e450 +Subproject commit 8f36d71d18587f1f315ec832f424183cb6519cbb From e737acb1f9fb08d0a187959b73dad2089d5404be Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:39:26 +0200 Subject: [PATCH 0956/1165] beautify tests --- ...2_storage_memory_modify_settings.reference | 6 ++-- .../03032_storage_memory_modify_settings.sql | 36 ++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference b/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference index f7d25c40a43..ca772aa7fd0 100644 --- a/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference +++ b/tests/queries/0_stateless/03032_storage_memory_modify_settings.reference @@ -3,9 +3,9 @@ TESTING MODIFY SMALLER BYTES 16384 65536 TESTING MODIFY SMALLER ROWS -9216 -8192 -4096 +1100 +1000 +500 TESTING ADD SETTINGS 50 1000 diff --git a/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql index bfa13ee0ec8..5a28af23fa7 100644 --- a/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql +++ b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql @@ -4,47 +4,47 @@ SELECT 'TESTING MODIFY SMALLER BYTES'; DROP TABLE IF EXISTS memory; CREATE TABLE memory (i UInt32) ENGINE = Memory SETTINGS min_bytes_to_keep = 8192, max_bytes_to_keep = 32768; -INSERT INTO memory SELECT * FROM numbers(0, 100); -INSERT INTO memory SELECT * FROM numbers(0, 3000); +INSERT INTO memory SELECT * FROM numbers(0, 100); -- 1024 bytes +INSERT INTO memory SELECT * FROM numbers(0, 3000); -- 16384 bytes SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); ALTER TABLE memory MODIFY SETTING min_bytes_to_keep = 4096, max_bytes_to_keep = 16384; SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(3000, 10000); +INSERT INTO memory SELECT * FROM numbers(3000, 10000); -- 65536 bytes SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); SELECT 'TESTING MODIFY SMALLER ROWS'; DROP TABLE IF EXISTS memory; CREATE TABLE memory (i UInt32) ENGINE = Memory SETTINGS min_rows_to_keep = 200, max_rows_to_keep = 2000; -INSERT INTO memory SELECT * FROM numbers(0, 100); -INSERT INTO memory SELECT * FROM numbers(100, 1000); -SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +INSERT INTO memory SELECT * FROM numbers(0, 100); -- 100 rows +INSERT INTO memory SELECT * FROM numbers(100, 1000); -- 1000 rows +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 1100 in total ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; -SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 1000 in total after deleting -INSERT INTO memory SELECT * FROM numbers(1000, 500); -SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +INSERT INTO memory SELECT * FROM numbers(1000, 500); -- 500 rows +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 500 in total after deleting SELECT 'TESTING ADD SETTINGS'; DROP TABLE IF EXISTS memory; CREATE TABLE memory (i UInt32) ENGINE = Memory; -INSERT INTO memory SELECT * FROM numbers(0, 50); +INSERT INTO memory SELECT * FROM numbers(0, 50); -- 50 rows SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(50, 950); +INSERT INTO memory SELECT * FROM numbers(50, 950); -- 950 rows SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(2000, 70); +INSERT INTO memory SELECT * FROM numbers(2000, 70); -- 70 rows SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(3000, 1100); +INSERT INTO memory SELECT * FROM numbers(3000, 1100); -- 1100 rows SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); SELECT 'TESTING ADD SETTINGS'; @@ -52,21 +52,23 @@ DROP TABLE IF EXISTS memory; CREATE TABLE memory (i UInt32) ENGINE = Memory; ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; -INSERT INTO memory SELECT * FROM numbers(0, 50); +INSERT INTO memory SELECT * FROM numbers(0, 50); -- 50 rows SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(50, 950); +INSERT INTO memory SELECT * FROM numbers(50, 950); -- 950 rows SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(2000, 70); +INSERT INTO memory SELECT * FROM numbers(2000, 70); -- 70 rows SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -INSERT INTO memory SELECT * FROM numbers(3000, 1100); +INSERT INTO memory SELECT * FROM numbers(3000, 1100); -- 1100 rows SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); SELECT 'TESTING INVALID SETTINGS'; DROP TABLE IF EXISTS memory; CREATE TABLE memory (i UInt32) ENGINE = Memory; +ALTER TABLE memory MODIFY SETTING max_rows_to_keep = 1000; +ALTER TABLE memory MODIFY SETTING max_bytes_to_keep = 1000; ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100; -- { serverError 452 } ALTER TABLE memory MODIFY SETTING min_bytes_to_keep = 100; -- { serverError 452 } From e0e3db843aeb32fba9fa0def4fb1bc7f17943bc5 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 3 Apr 2024 18:03:45 +0200 Subject: [PATCH 0957/1165] Update 03032_storage_memory_modify_settings.sql --- .../0_stateless/03032_storage_memory_modify_settings.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql index 5a28af23fa7..e59a5e4edb6 100644 --- a/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql +++ b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql @@ -67,9 +67,9 @@ SELECT total_rows FROM system.tables WHERE name = 'memory' and database = curren SELECT 'TESTING INVALID SETTINGS'; DROP TABLE IF EXISTS memory; CREATE TABLE memory (i UInt32) ENGINE = Memory; -ALTER TABLE memory MODIFY SETTING max_rows_to_keep = 1000; -ALTER TABLE memory MODIFY SETTING max_bytes_to_keep = 1000; ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100; -- { serverError 452 } ALTER TABLE memory MODIFY SETTING min_bytes_to_keep = 100; -- { serverError 452 } +ALTER TABLE memory MODIFY SETTING max_rows_to_keep = 1000; +ALTER TABLE memory MODIFY SETTING max_bytes_to_keep = 1000; -DROP TABLE memory; \ No newline at end of file +DROP TABLE memory; From ca27cf3fde37442f76f3d244f8cc57b5c541ebc7 Mon Sep 17 00:00:00 2001 From: yariks5s Date: Wed, 3 Apr 2024 16:38:41 +0000 Subject: [PATCH 0958/1165] fix mapper for gcs --- programs/server/config.xml | 2 +- src/IO/S3/URI.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index ea3ead47c32..e92381eeb1e 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -96,7 +96,7 @@ https://{bucket}.s3.amazonaws.com - https://{bucket}.storage.googleapis.com + https://storage.googleapis.com/{bucket} https://{bucket}.oss.aliyuncs.com diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 027cb624ed5..0d8502ecf1f 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -67,7 +67,7 @@ URI::URI(const std::string & uri_) else { mapper["s3"] = "https://{bucket}.s3.amazonaws.com"; - mapper["gs"] = "https://{bucket}.storage.googleapis.com"; + mapper["gs"] = "https://storage.googleapis.com/{bucket}"; mapper["oss"] = "https://{bucket}.oss.aliyuncs.com"; } From 463691922c43a9ee30ad7d0a3a7895fcbd76760a Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 3 Apr 2024 17:05:49 +0000 Subject: [PATCH 0959/1165] Fix: disable test for SMT --- .../0_stateless/02980_dist_insert_readonly_replica.sql.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02980_dist_insert_readonly_replica.sql.j2 b/tests/queries/0_stateless/02980_dist_insert_readonly_replica.sql.j2 index 5bf40f34f5c..aba742fa64a 100644 --- a/tests/queries/0_stateless/02980_dist_insert_readonly_replica.sql.j2 +++ b/tests/queries/0_stateless/02980_dist_insert_readonly_replica.sql.j2 @@ -1,6 +1,7 @@ --- Tags: no-parallel, no-fasttest +-- Tags: no-parallel, no-fasttest, no-shared-merge-tree -- Tag no-parallel - due to static databases -- Tag no-fasttest - S3 is required +-- Tag no-shared-merge-tree - no reliable way to make SMT read-only in stateless test drop database if exists shard_0; drop database if exists shard_1; From 2db1e3451773c160382325334a6c2b992c8b3314 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 3 Apr 2024 19:06:33 +0200 Subject: [PATCH 0960/1165] Better logger name --- src/Interpreters/Cache/FileCache.cpp | 10 ++++++++-- src/Interpreters/Cache/SLRUFileCachePriority.cpp | 8 +++++--- src/Interpreters/Cache/SLRUFileCachePriority.h | 5 +++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 71dc0cca3a7..3dc13ba7737 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -90,9 +90,15 @@ FileCache::FileCache(const std::string & cache_name, const FileCacheSettings & s , metadata(settings.base_path, settings.background_download_queue_size_limit, settings.background_download_threads, write_cache_per_user_directory) { if (settings.cache_policy == "LRU") - main_priority = std::make_unique(settings.max_size, settings.max_elements); + { + main_priority = std::make_unique( + settings.max_size, settings.max_elements, nullptr, cache_name); + } else if (settings.cache_policy == "SLRU") - main_priority = std::make_unique(settings.max_size, settings.max_elements, settings.slru_size_ratio); + { + main_priority = std::make_unique( + settings.max_size, settings.max_elements, settings.slru_size_ratio, nullptr, nullptr, cache_name); + } else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown cache policy: {}", settings.cache_policy); diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 1400d3219c6..59e51ae31f1 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -28,17 +28,19 @@ SLRUFileCachePriority::SLRUFileCachePriority( size_t max_elements_, double size_ratio_, LRUFileCachePriority::StatePtr probationary_state_, - LRUFileCachePriority::StatePtr protected_state_) + LRUFileCachePriority::StatePtr protected_state_, + const std::string & description_) : IFileCachePriority(max_size_, max_elements_) , size_ratio(size_ratio_) , protected_queue(LRUFileCachePriority(getRatio(max_size_, size_ratio), getRatio(max_elements_, size_ratio), protected_state_, - "protected")) + description_ + ", protected")) , probationary_queue(LRUFileCachePriority(getRatio(max_size_, 1 - size_ratio), getRatio(max_elements_, 1 - size_ratio), probationary_state_, - "probationary")) + description_ + ", probationary")) + , log(getLogger("SLRUFileCachePriority(" + description_ + ")")) { LOG_DEBUG( log, "Probationary queue {} in size and {} in elements. " diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h index 4cf5bb0f199..734828f55dd 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.h +++ b/src/Interpreters/Cache/SLRUFileCachePriority.h @@ -19,7 +19,8 @@ public: size_t max_elements_, double size_ratio_, LRUFileCachePriority::StatePtr probationary_state_ = nullptr, - LRUFileCachePriority::StatePtr protected_state_ = nullptr); + LRUFileCachePriority::StatePtr protected_state_ = nullptr, + const std::string & description_ = "none"); size_t getSize(const CachePriorityGuard::Lock & lock) const override; @@ -67,7 +68,7 @@ private: double size_ratio; LRUFileCachePriority protected_queue; LRUFileCachePriority probationary_queue; - LoggerPtr log = getLogger("SLRUFileCachePriority"); + LoggerPtr log; void increasePriority(SLRUIterator & iterator, const CachePriorityGuard::Lock & lock); From 11eb872ac61a94bce68e715f9f855d59fd4aadd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 3 Apr 2024 19:06:36 +0200 Subject: [PATCH 0961/1165] Remove more nested includes --- src/Access/AccessBackup.cpp | 2 ++ src/Backups/BackupEntriesCollector.cpp | 11 +++++++---- src/Backups/BackupsWorker.cpp | 2 ++ src/Backups/RestorerFromBackup.cpp | 3 +++ src/Columns/ColumnAggregateFunction.cpp | 9 +++++---- src/Columns/ColumnAggregateFunction.h | 9 +-------- src/Columns/ColumnObject.h | 5 ++--- src/Common/FileChecker.cpp | 2 ++ src/Core/Field.cpp | 12 +++++++++++- src/Core/Field.h | 9 +++++++-- src/Core/SettingsEnums.cpp | 2 ++ src/Core/SettingsFields.h | 11 +++++------ .../SerializationAggregateFunction.cpp | 17 +++++++---------- src/Functions/FunctionsTimeWindow.h | 1 + src/Functions/array/mapOp.cpp | 7 ++++--- src/Interpreters/DatabaseCatalog.cpp | 2 ++ src/Storages/StorageLog.cpp | 2 ++ src/Storages/System/StorageSystemTables.cpp | 2 ++ .../TableFunctionMergeTreeIndex.cpp | 2 ++ 19 files changed, 69 insertions(+), 41 deletions(-) diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index ba89899dd8f..1110b9c4b21 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -16,6 +16,8 @@ #include #include #include + +#include #include namespace fs = std::filesystem; diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index c71ce195388..2efb6bf84ad 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -1,22 +1,25 @@ +#include +#include #include #include -#include -#include #include #include +#include #include #include #include #include #include #include -#include #include #include +#include #include #include + +#include #include -#include + #include namespace fs = std::filesystem; diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 96fe770227c..5cbc4ed1e41 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -27,6 +27,8 @@ #include #include +#include + namespace CurrentMetrics { diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index ed1d5b8a103..d8383fdbb1d 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -24,6 +24,9 @@ #include #include #include + +#include + #include #include diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index 801aa8a91bb..7533d2cb71b 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -2,6 +2,11 @@ #include #include +#include +#include +#include +#include +#include #include #include #include @@ -11,10 +16,6 @@ #include #include #include -#include -#include -#include -#include namespace DB diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index ae7c5f0b54e..ba33cf0ee14 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -1,17 +1,10 @@ #pragma once #include - #include -#include - #include - -#include -#include -#include - #include +#include namespace DB { diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index a06235a4500..af12b99efcb 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -1,12 +1,11 @@ #pragma once +#include #include #include -#include -#include -#include #include #include +#include #include diff --git a/src/Common/FileChecker.cpp b/src/Common/FileChecker.cpp index 098ea4b1ac4..5ecbe44530b 100644 --- a/src/Common/FileChecker.cpp +++ b/src/Common/FileChecker.cpp @@ -10,6 +10,8 @@ #include #include +#include + namespace fs = std::filesystem; diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 9c058d61902..73f0703f21e 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -22,6 +22,12 @@ namespace ErrorCodes extern const int DECIMAL_OVERFLOW; } +template +T DecimalField::getScaleMultiplier() const +{ + return DecimalUtils::scaleMultiplier(scale); +} + inline Field getBinaryValue(UInt8 type, ReadBuffer & buf) { switch (static_cast(type)) @@ -627,5 +633,9 @@ std::string_view Field::getTypeName() const return fieldTypeToString(which); } - +template class DecimalField; +template class DecimalField; +template class DecimalField; +template class DecimalField; +template class DecimalField; } diff --git a/src/Core/Field.h b/src/Core/Field.h index aed5fab2106..eb01be6c43d 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -8,7 +8,6 @@ #include #include -#include #include #include #include @@ -151,7 +150,7 @@ public: operator T() const { return dec; } /// NOLINT T getValue() const { return dec; } - T getScaleMultiplier() const { return DecimalUtils::scaleMultiplier(scale); } + T getScaleMultiplier() const; UInt32 getScale() const { return scale; } template @@ -200,6 +199,12 @@ private: UInt32 scale; }; +extern template class DecimalField; +extern template class DecimalField; +extern template class DecimalField; +extern template class DecimalField; +extern template class DecimalField; + template constexpr bool is_decimal_field = false; template <> constexpr inline bool is_decimal_field> = true; template <> constexpr inline bool is_decimal_field> = true; diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index c3f0715ad68..e47174a3b9d 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -2,6 +2,8 @@ #include #include +#include + namespace DB { diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index b39aa52c15d..64854e46ab5 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -1,13 +1,12 @@ #pragma once -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include namespace DB diff --git a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp index 6f09ed31e22..2ac23d52e28 100644 --- a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp +++ b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp @@ -1,17 +1,14 @@ -#include - -#include - #include - -#include -#include -#include -#include - +#include #include #include +#include #include +#include +#include +#include +#include +#include namespace DB { diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index 4532286830d..6183d25c8bd 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include diff --git a/src/Functions/array/mapOp.cpp b/src/Functions/array/mapOp.cpp index 613fd934c41..50b64cf9809 100644 --- a/src/Functions/array/mapOp.cpp +++ b/src/Functions/array/mapOp.cpp @@ -1,18 +1,19 @@ -#include #include +#include #include #include #include #include #include #include +#include #include #include #include #include -#include "Columns/ColumnMap.h" -#include "DataTypes/DataTypeMap.h" +#include +#include namespace DB { diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index ec6c8b5924f..2536ce0ce2f 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -27,6 +27,8 @@ #include #include +#include + #include "config.h" #if USE_MYSQL diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 549cfca1b6c..b652750346f 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -39,6 +39,8 @@ #include #include +#include + #define DBMS_STORAGE_LOG_DATA_FILE_EXTENSION ".bin" #define DBMS_STORAGE_LOG_MARKS_FILE_NAME "__marks.mrk" diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index edfc7213dcd..9bd7ff945ad 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -25,6 +25,8 @@ #include #include +#include + namespace DB { diff --git a/src/TableFunctions/TableFunctionMergeTreeIndex.cpp b/src/TableFunctions/TableFunctionMergeTreeIndex.cpp index 435ed4bdf0d..06a48f0e25f 100644 --- a/src/TableFunctions/TableFunctionMergeTreeIndex.cpp +++ b/src/TableFunctions/TableFunctionMergeTreeIndex.cpp @@ -10,6 +10,8 @@ #include #include +#include + namespace DB { From c70ecfe5f3bc37ad49dffb79158a86dc887de798 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 3 Apr 2024 19:18:56 +0200 Subject: [PATCH 0962/1165] Fix --- .../Cache/LRUFileCachePriority.cpp | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index ddc30755409..012ba14e5a8 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -307,25 +307,29 @@ bool LRUFileCachePriority::collectCandidatesForEviction( if (can_fit()) { - /// As eviction is done without a cache priority lock, - /// then if some space was partially available and some needed - /// to be freed via eviction, we need to make sure that this - /// partially available space is still available - /// after we finish with eviction for non-available space. - /// So we create a space holder for the currently available part - /// of the required space for the duration of eviction of the other - /// currently non-available part of the space. + /// `res` contains eviction candidates. Do we have any? + if (res.size() > 0) + { + /// As eviction is done without a cache priority lock, + /// then if some space was partially available and some needed + /// to be freed via eviction, we need to make sure that this + /// partially available space is still available + /// after we finish with eviction for non-available space. + /// So we create a space holder for the currently available part + /// of the required space for the duration of eviction of the other + /// currently non-available part of the space. - const size_t hold_size = size > stat.total_stat.releasable_size - ? size - stat.total_stat.releasable_size - : 0; + const size_t hold_size = size > stat.total_stat.releasable_size + ? size - stat.total_stat.releasable_size + : 0; - const size_t hold_elements = elements > stat.total_stat.releasable_count - ? elements - stat.total_stat.releasable_count - : 0; + const size_t hold_elements = elements > stat.total_stat.releasable_count + ? elements - stat.total_stat.releasable_count + : 0; - if (hold_size || hold_elements) - res.setSpaceHolder(hold_size, hold_elements, *this, lock); + if (hold_size || hold_elements) + res.setSpaceHolder(hold_size, hold_elements, *this, lock); + } // LOG_TEST(log, "Collected {} candidates for eviction (total size: {}). " // "Took hold of size {} and elements {}", From 0dbb249b4878f168fbcc4730437911f50158b6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 3 Apr 2024 19:30:55 +0200 Subject: [PATCH 0963/1165] More nested includes --- src/Columns/ColumnAggregateFunction.cpp | 6 ++++++ src/Columns/ColumnAggregateFunction.h | 13 +++++++++++-- src/Columns/ColumnObject.h | 1 - src/Core/Block.cpp | 20 +++++++++----------- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index 7533d2cb71b..f7e6b1a1ccc 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -110,6 +111,11 @@ ConstArenas concatArenas(const ConstArenas & array, ConstArenaPtr arena) } +std::string ColumnAggregateFunction::getName() const +{ + return "AggregateFunction(" + func->getName() + ")"; +} + MutableColumnPtr ColumnAggregateFunction::convertToValues(MutableColumnPtr column) { /** If the aggregate function returns an unfinalized/unfinished state, diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index ba33cf0ee14..3c836f50bdf 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -19,6 +18,16 @@ using ArenaPtr = std::shared_ptr; using ConstArenaPtr = std::shared_ptr; using ConstArenas = std::vector; +using AggregateDataPtr = char *; +using AggregateDataPtrs = std::vector; +using ConstAggregateDataPtr = const char *; + +class IAggregateFunction; +using AggregateFunctionPtr = std::shared_ptr; + +class Context; +using ContextPtr = std::shared_ptr; + /** Column of states of aggregate functions. * Presented as an array of pointers to the states of aggregate functions (data). @@ -114,7 +123,7 @@ public: /// This method is made static and receive MutableColumnPtr object to explicitly destroy it. static MutableColumnPtr convertToValues(MutableColumnPtr column); - std::string getName() const override { return "AggregateFunction(" + func->getName() + ")"; } + std::string getName() const override; const char * getFamilyName() const override { return "AggregateFunction"; } TypeIndex getDataType() const override { return TypeIndex::AggregateFunction; } diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index af12b99efcb..e2936b27994 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index dfd60b994f4..77dbad5443e 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -1,19 +1,17 @@ -#include -#include - -#include - -#include -#include - -#include - +#include #include #include #include +#include +#include +#include +#include +#include +#include +#include #include -#include + #include From 40b901a2e3f37db01b899319aba75126c1047bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 3 Apr 2024 19:49:56 +0200 Subject: [PATCH 0964/1165] More crap --- src/Columns/ColumnAggregateFunction.h | 4 +- src/Interpreters/Aggregator.cpp | 47 +++++++++---------- src/Interpreters/InterpreterSelectQuery.cpp | 1 + src/Planner/Planner.cpp | 7 +-- src/Processors/QueryPlan/AggregatingStep.cpp | 1 + src/Processors/Transforms/WindowTransform.cpp | 32 ++++++------- 6 files changed, 48 insertions(+), 44 deletions(-) diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 3c836f50bdf..c5d854e208a 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -2,7 +2,6 @@ #include #include -#include #include namespace DB @@ -28,6 +27,9 @@ using AggregateFunctionPtr = std::shared_ptr; class Context; using ContextPtr = std::shared_ptr; +struct ColumnWithTypeAndName; +using ColumnsWithTypeAndName = std::vector; + /** Column of states of aggregate functions. * Presented as an array of pointers to the states of aggregate functions (data). diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index ab8cec864ae..bd78c7a1bc1 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -7,38 +7,37 @@ # include #endif -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include #include -#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include #include - -#include - -#include +#include +#include namespace ProfileEvents diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c47e3bdc49f..d518c8d96b4 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 5f73bba67a6..d75573c8d99 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1,9 +1,10 @@ #include -#include -#include -#include +#include #include +#include +#include +#include #include diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 74f293e5682..0d7e05af1de 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 02d2762dab3..f43b9a2e794 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -1,23 +1,23 @@ -#include - -#include - #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include + +#include /// See https://fmt.dev/latest/api.html#formatting-user-defined-types From 925148f928c3d6165a5f8b217ca8a597b0962ab1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 3 Apr 2024 20:32:48 +0200 Subject: [PATCH 0965/1165] Add logging --- src/Interpreters/Cache/FileCache.cpp | 1 - src/Interpreters/Cache/FileCacheFactory.cpp | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 75e199c544b..90671629e64 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1379,7 +1379,6 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, } } - if (new_settings.max_size != actual_settings.max_size || new_settings.max_elements != actual_settings.max_elements) { diff --git a/src/Interpreters/Cache/FileCacheFactory.cpp b/src/Interpreters/Cache/FileCacheFactory.cpp index e05e1935d95..e1f14406058 100644 --- a/src/Interpreters/Cache/FileCacheFactory.cpp +++ b/src/Interpreters/Cache/FileCacheFactory.cpp @@ -142,8 +142,10 @@ void FileCacheFactory::updateSettingsFromConfig(const Poco::Util::AbstractConfig caches_by_name_copy = caches_by_name; } + auto * log = &Poco::Logger::get("FileCacheFactory"); + std::unordered_set checked_paths; - for (const auto & [_, cache_info] : caches_by_name_copy) + for (const auto & [cache_name, cache_info] : caches_by_name_copy) { if (cache_info->config_path.empty() || checked_paths.contains(cache_info->config_path)) continue; @@ -155,7 +157,12 @@ void FileCacheFactory::updateSettingsFromConfig(const Poco::Util::AbstractConfig FileCacheSettings old_settings = cache_info->getSettings(); if (old_settings == new_settings) + { + LOG_TRACE(log, "No settings changes for cache: {}", cache_name); continue; + } + + LOG_TRACE(log, "Will apply settings changes for cache: {}", cache_name); try { @@ -166,6 +173,7 @@ void FileCacheFactory::updateSettingsFromConfig(const Poco::Util::AbstractConfig /// Settings changes could be partially applied in case of exception, /// make sure cache_info->settings show correct state of applied settings. cache_info->setSettings(old_settings); + tryLogCurrentException(__PRETTY_FUNCTION__); throw; } From 76b6c125ff136637437e200f4546c838f90f5bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 3 Apr 2024 20:36:29 +0200 Subject: [PATCH 0966/1165] Remove boost dep in wide integers when possible --- base/base/wide_integer_impl.h | 11 +++++++++-- src/IO/ReadBufferFromIStream.cpp | 2 ++ src/IO/SeekableReadBuffer.cpp | 1 + src/IO/VarInt.h | 3 +++ src/IO/WriteBufferFromOStream.cpp | 1 + 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 17b1fa7cd6a..0e98b6e5ee6 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -13,8 +13,6 @@ #include #include -#include - // NOLINTBEGIN(*) /// Use same extended double for all platforms @@ -22,6 +20,7 @@ #define CONSTEXPR_FROM_DOUBLE constexpr using FromDoubleIntermediateType = long double; #else +#include #include /// `wide_integer_from_builtin` can't be constexpr with non-literal `cpp_bin_float_double_extended` #define CONSTEXPR_FROM_DOUBLE @@ -309,6 +308,13 @@ struct integer::_impl constexpr uint64_t max_int = std::numeric_limits::max(); static_assert(std::is_same_v || std::is_same_v); /// Implementation specific behaviour on overflow (if we don't check here, stack overflow will triggered in bigint_cast). +#if (LDBL_MANT_DIG == 64) + if (!std::isfinite(t)) + { + self = 0; + return; + } +#else if constexpr (std::is_same_v) { if (!std::isfinite(t)) @@ -325,6 +331,7 @@ struct integer::_impl return; } } +#endif const T alpha = t / static_cast(max_int); diff --git a/src/IO/ReadBufferFromIStream.cpp b/src/IO/ReadBufferFromIStream.cpp index bc90ec7ed15..325beabaf81 100644 --- a/src/IO/ReadBufferFromIStream.cpp +++ b/src/IO/ReadBufferFromIStream.cpp @@ -1,6 +1,8 @@ #include #include +#include + namespace DB { diff --git a/src/IO/SeekableReadBuffer.cpp b/src/IO/SeekableReadBuffer.cpp index 5d83f4e1b4a..f2a114a5389 100644 --- a/src/IO/SeekableReadBuffer.cpp +++ b/src/IO/SeekableReadBuffer.cpp @@ -1,5 +1,6 @@ #include +#include namespace DB { diff --git a/src/IO/VarInt.h b/src/IO/VarInt.h index 9e72705341d..6dce8008170 100644 --- a/src/IO/VarInt.h +++ b/src/IO/VarInt.h @@ -5,6 +5,9 @@ #include #include +#include +#include + namespace DB { diff --git a/src/IO/WriteBufferFromOStream.cpp b/src/IO/WriteBufferFromOStream.cpp index ffc3e62e9a6..e77ec079d1f 100644 --- a/src/IO/WriteBufferFromOStream.cpp +++ b/src/IO/WriteBufferFromOStream.cpp @@ -1,6 +1,7 @@ #include #include +#include namespace DB { From be55c3533c9b3b4293bbb052dc8ed585af7c20b7 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 3 Apr 2024 15:48:30 -0300 Subject: [PATCH 0967/1165] default to path style --- src/IO/S3/URI.cpp | 15 +++------------ src/IO/tests/gtest_s3_uri.cpp | 6 +++--- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 8e364337db5..02c77518ab6 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -108,19 +108,10 @@ URI::URI(const std::string & uri_) String name; String endpoint_authority_from_uri; - if (re2::RE2::FullMatch(uri.getAuthority(), aws_private_link_style_pattern)) - { - if (!re2::RE2::PartialMatch(uri.getPath(), path_style_pattern, &bucket, &key)) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Could not parse bucket and key from uri {}", uri.toString()); - } + bool is_using_aws_private_link_interface = re2::RE2::FullMatch(uri.getAuthority(), aws_private_link_style_pattern); - // Default to virtual hosted style - is_virtual_hosted_style = true; - endpoint = uri.getScheme() + "://" + uri.getAuthority(); - validateBucket(bucket, uri); - } - else if (re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &name, &endpoint_authority_from_uri)) + if (!is_using_aws_private_link_interface + && re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &name, &endpoint_authority_from_uri)) { is_virtual_hosted_style = true; endpoint = uri.getScheme() + "://" + name + endpoint_authority_from_uri; diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index b3ceb875362..0a164b0dd61 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -79,21 +79,21 @@ const TestCase TestCases[] = { "root", "nested/file.txt", "", - true}, + false}, // Test with a file with no extension {S3::URI("https://bucket.vpce-03b2c987f1bd55c5f-j3b4vg7w.s3.ap-southeast-2.vpce.amazonaws.com/some_bucket/document"), "https://bucket.vpce-03b2c987f1bd55c5f-j3b4vg7w.s3.ap-southeast-2.vpce.amazonaws.com", "some_bucket", "document", "", - true}, + false}, // Test with a deeply nested file path {S3::URI("https://bucket.vpce-0242cd56f1bd55c5f-l5b7vg8x.s3.sa-east-1.vpce.amazonaws.com/some_bucket/b/c/d/e/f/g/h/i/j/data.json"), "https://bucket.vpce-0242cd56f1bd55c5f-l5b7vg8x.s3.sa-east-1.vpce.amazonaws.com", "some_bucket", "b/c/d/e/f/g/h/i/j/data.json", "", - true}, + false}, }; class S3UriTest : public testing::TestWithParam From de2a0be02580eb7501ad1fec5de35a7107ef9a1e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 3 Apr 2024 18:50:33 +0000 Subject: [PATCH 0968/1165] Don't access static members through instance - clang-tidy rightfully complains (-readability-static-accessed-through-instance) - not going to enable the warning for now to avoid breaking the build --- programs/format/Format.cpp | 2 +- .../ExternalDictionaryLibraryUtils.h | 2 +- .../AggregateFunctionAnyHeavy.cpp | 12 ++++++------ .../AggregateFunctionFlameGraph.cpp | 6 +++--- .../AggregateFunctionGroupArray.cpp | 4 ++-- ...gateFunctionLargestTriangleThreeBuckets.cpp | 12 ++++++------ .../AggregateFunctionMannWhitney.cpp | 16 ++++++++-------- .../AggregateFunctionRankCorrelation.cpp | 14 +++++++------- .../AggregateFunctionRetention.cpp | 12 ++++++------ ...AggregateFunctionSimpleLinearRegression.cpp | 12 ++++++------ .../AggregateFunctionSingleValueOrNull.cpp | 16 ++++++++-------- .../AggregateFunctionStatistics.cpp | 18 +++++++++--------- .../Combinators/AggregateFunctionNull.h | 2 +- src/AggregateFunctions/ReservoirSampler.h | 4 ++-- src/Analyzer/Passes/QueryAnalysisPass.cpp | 3 +-- src/Columns/ColumnSparse.cpp | 2 +- src/Common/AsynchronousMetrics.cpp | 2 +- src/Common/HTTPConnectionPool.cpp | 2 +- src/Common/HashTable/HashMap.h | 6 +++--- src/Common/StackTrace.cpp | 2 +- src/Common/ThreadStatus.cpp | 2 +- src/Compression/CachedCompressedReadBuffer.cpp | 2 +- src/Databases/DatabaseOnDisk.cpp | 2 +- src/Databases/DatabaseOrdinary.cpp | 2 +- src/Databases/TablesDependencyGraph.cpp | 2 +- src/Dictionaries/FlatDictionary.cpp | 3 +-- src/Dictionaries/PolygonDictionaryUtils.h | 2 +- src/Disks/DiskLocal.cpp | 2 +- src/Functions/FunctionsExternalDictionaries.h | 6 +++--- .../UserDefinedSQLObjectsStorageBase.cpp | 2 +- src/IO/Archives/LibArchiveWriter.h | 2 +- src/IO/MMapReadBufferFromFileWithCache.cpp | 2 +- src/IO/ReadHelpers.h | 10 +++++----- src/IO/WriteHelpers.h | 4 ++-- src/Interpreters/AsynchronousInsertQueue.cpp | 2 +- .../ExecuteScalarSubqueriesVisitor.cpp | 2 +- .../InterpreterCreateFunctionQuery.cpp | 2 +- src/Interpreters/InterpreterDeleteQuery.cpp | 2 +- .../InterpreterDropFunctionQuery.cpp | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 10 +++++----- src/Interpreters/TreeRewriter.cpp | 2 +- .../evaluateConstantExpression.cpp | 2 +- src/Planner/PlannerActionsVisitor.cpp | 2 +- src/Planner/PlannerJoinTree.cpp | 2 +- src/Processors/Executors/ExecutorTasks.cpp | 2 +- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 4 ++-- src/Server/HTTP/ReadHeaders.cpp | 2 +- src/Storages/AlterCommands.cpp | 2 +- src/Storages/FileLog/StorageFileLog.h | 2 +- src/Storages/KeyDescription.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 16 ++++++++-------- src/Storages/MergeTree/KeyCondition.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeDataWriter.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeSink.cpp | 2 +- .../MergeTree/MutateFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MutateTask.cpp | 12 ++++++------ .../MergeTree/ReplicatedMergeTreeSink.cpp | 2 +- .../ReplicatedMergeTreeTableMetadata.cpp | 2 +- .../MergeTree/registerStorageMergeTree.cpp | 4 ++-- src/Storages/StorageFile.cpp | 2 +- src/Storages/StorageFile.h | 2 +- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/StorageS3.cpp | 2 +- 66 files changed, 149 insertions(+), 151 deletions(-) diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index fc73eda6815..d4b975ce1e8 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -237,7 +237,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ASTPtr res = parseQueryAndMovePosition( parser, pos, end, "query", multiple, cmd_settings.max_query_size, cmd_settings.max_parser_depth, cmd_settings.max_parser_backtracks); - std::unique_ptr insert_query_payload = nullptr; + std::unique_ptr insert_query_payload; /// If the query is INSERT ... VALUES, then we will try to parse the data. if (auto * insert_query = res->as(); insert_query && insert_query->data) { diff --git a/programs/library-bridge/ExternalDictionaryLibraryUtils.h b/programs/library-bridge/ExternalDictionaryLibraryUtils.h index c9d03d27f75..e6bf8f2a4c3 100644 --- a/programs/library-bridge/ExternalDictionaryLibraryUtils.h +++ b/programs/library-bridge/ExternalDictionaryLibraryUtils.h @@ -35,7 +35,7 @@ public: ExternalDictionaryLibraryAPI::CStrings strings; // will pass pointer to lib private: - std::unique_ptr ptr_holder = nullptr; + std::unique_ptr ptr_holder; Container strings_holder; }; diff --git a/src/AggregateFunctions/AggregateFunctionAnyHeavy.cpp b/src/AggregateFunctions/AggregateFunctionAnyHeavy.cpp index 4f4d4a19cba..ffddd46f2e3 100644 --- a/src/AggregateFunctions/AggregateFunctionAnyHeavy.cpp +++ b/src/AggregateFunctions/AggregateFunctionAnyHeavy.cpp @@ -115,34 +115,34 @@ public: void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { - this->data(place).add(*columns[0], row_num, arena); + data(place).add(*columns[0], row_num, arena); } void addManyDefaults(AggregateDataPtr __restrict place, const IColumn ** columns, size_t, Arena * arena) const override { - this->data(place).addManyDefaults(*columns[0], 0, arena); + data(place).addManyDefaults(*columns[0], 0, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - this->data(place).add(this->data(rhs), arena); + data(place).add(data(rhs), arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf, *serialization); + data(place).write(buf, *serialization); } void deserialize(AggregateDataPtr place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, *serialization, arena); + data(place).read(buf, *serialization, arena); } bool allocatesMemoryInArena() const override { return singleValueTypeAllocatesMemoryInArena(value_type_index); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - this->data(place).insertResultInto(to); + data(place).insertResultInto(to); } }; diff --git a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp index f3d99046036..33e318b6c2f 100644 --- a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp +++ b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp @@ -559,7 +559,7 @@ public: ptr = ptrs[row_num]; } - this->data(place).add(ptr, allocated, trace_values.data() + prev_offset, trace_size, arena); + data(place).add(ptr, allocated, trace_values.data() + prev_offset, trace_size, arena); } void addManyDefaults( @@ -572,7 +572,7 @@ public: void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - this->data(place).merge(this->data(rhs), arena); + data(place).merge(data(rhs), arena); } void serialize(ConstAggregateDataPtr __restrict, WriteBuffer &, std::optional /* version */) const override @@ -590,7 +590,7 @@ public: auto & array = assert_cast(to); auto & str = assert_cast(array.getData()); - this->data(place).dumpFlameGraph(str.getChars(), str.getOffsets(), 0, 0); + data(place).dumpFlameGraph(str.getChars(), str.getOffsets(), 0, 0); array.getOffsets().push_back(str.size()); } diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 6af8b1018dd..63002652166 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -89,10 +89,10 @@ struct GroupArraySamplerData chassert(lim != 0); /// With a large number of values, we will generate random numbers several times slower. - if (lim <= static_cast(rng.max())) + if (lim <= static_cast(pcg32_fast::max())) return rng() % lim; else - return (static_cast(rng()) * (static_cast(rng.max()) + 1ULL) + static_cast(rng())) % lim; + return (static_cast(rng()) * (static_cast(pcg32::max()) + 1ULL) + static_cast(rng())) % lim; } void randomShuffle() diff --git a/src/AggregateFunctions/AggregateFunctionLargestTriangleThreeBuckets.cpp b/src/AggregateFunctions/AggregateFunctionLargestTriangleThreeBuckets.cpp index d5abdbc12fb..b24b6c8996f 100644 --- a/src/AggregateFunctions/AggregateFunctionLargestTriangleThreeBuckets.cpp +++ b/src/AggregateFunctions/AggregateFunctionLargestTriangleThreeBuckets.cpp @@ -242,7 +242,7 @@ public: { Float64 x = getFloat64DataFromColumn(columns[0], row_num, this->x_type); Float64 y = getFloat64DataFromColumn(columns[1], row_num, this->y_type); - this->data(place).add(x, y, arena); + data(place).add(x, y, arena); } Float64 getFloat64DataFromColumn(const IColumn * column, size_t row_num, TypeIndex type_index) const @@ -264,25 +264,25 @@ public: void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - auto & a = this->data(place); - const auto & b = this->data(rhs); + auto & a = data(place); + const auto & b = data(rhs); a.merge(b, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf); + data(place).write(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, arena); + data(place).read(buf, arena); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const override { - auto res = this->data(place).getResult(total_buckets, arena); + auto res = data(place).getResult(total_buckets, arena); auto & col = assert_cast(to); auto & col_offsets = assert_cast(col.getOffsetsColumn()); diff --git a/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp b/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp index a70da7b35d5..e7bc5df335f 100644 --- a/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp +++ b/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp @@ -205,35 +205,35 @@ public: UInt8 is_second = columns[1]->getUInt(row_num); if (is_second) - this->data(place).addY(value, arena); + data(place).addY(value, arena); else - this->data(place).addX(value, arena); + data(place).addX(value, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - auto & a = this->data(place); - const auto & b = this->data(rhs); + auto & a = data(place); + const auto & b = data(rhs); a.merge(b, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf); + data(place).write(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, arena); + data(place).read(buf, arena); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - if (!this->data(place).size_x || !this->data(place).size_y) + if (!data(place).size_x || !data(place).size_y) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} require both samples to be non empty", getName()); - auto [u_statistic, p_value] = this->data(place).getResult(alternative, continuity_correction); + auto [u_statistic, p_value] = data(place).getResult(alternative, continuity_correction); /// Because p-value is a probability. p_value = std::min(1.0, std::max(0.0, p_value)); diff --git a/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp b/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp index d338808c717..0c4726734ce 100644 --- a/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp +++ b/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp @@ -66,31 +66,31 @@ public: { Float64 new_x = columns[0]->getFloat64(row_num); Float64 new_y = columns[1]->getFloat64(row_num); - this->data(place).addX(new_x, arena); - this->data(place).addY(new_y, arena); + data(place).addX(new_x, arena); + data(place).addY(new_y, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - auto & a = this->data(place); - const auto & b = this->data(rhs); + auto & a = data(place); + const auto & b = data(rhs); a.merge(b, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf); + data(place).write(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, arena); + data(place).read(buf, arena); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - auto answer = this->data(place).getResult(); + auto answer = data(place).getResult(); auto & column = static_cast &>(to); column.getData().push_back(answer); diff --git a/src/AggregateFunctions/AggregateFunctionRetention.cpp b/src/AggregateFunctions/AggregateFunctionRetention.cpp index 5eaa1a7a39c..e9b46e62c14 100644 --- a/src/AggregateFunctions/AggregateFunctionRetention.cpp +++ b/src/AggregateFunctions/AggregateFunctionRetention.cpp @@ -102,24 +102,24 @@ public: auto event = assert_cast *>(columns[i])->getData()[row_num]; if (event) { - this->data(place).add(i); + data(place).add(i); } } } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override { - this->data(place).merge(this->data(rhs)); + data(place).merge(data(rhs)); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).serialize(buf); + data(place).serialize(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override { - this->data(place).deserialize(buf); + data(place).deserialize(buf); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override @@ -130,13 +130,13 @@ public: ColumnArray::Offset current_offset = data_to.size(); data_to.resize(current_offset + events_size); - const bool first_flag = this->data(place).events.test(0); + const bool first_flag = data(place).events.test(0); data_to[current_offset] = first_flag; ++current_offset; for (size_t i = 1; i < events_size; ++i) { - data_to[current_offset] = (first_flag && this->data(place).events.test(i)); + data_to[current_offset] = (first_flag && data(place).events.test(i)); ++current_offset; } diff --git a/src/AggregateFunctions/AggregateFunctionSimpleLinearRegression.cpp b/src/AggregateFunctions/AggregateFunctionSimpleLinearRegression.cpp index 75d2fe595d8..ce2f7ee195d 100644 --- a/src/AggregateFunctions/AggregateFunctionSimpleLinearRegression.cpp +++ b/src/AggregateFunctions/AggregateFunctionSimpleLinearRegression.cpp @@ -123,22 +123,22 @@ public: Float64 x = columns[0]->getFloat64(row_num); Float64 y = columns[1]->getFloat64(row_num); - this->data(place).add(x, y); + data(place).add(x, y); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override { - this->data(place).merge(this->data(rhs)); + data(place).merge(data(rhs)); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).serialize(buf); + data(place).serialize(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override { - this->data(place).deserialize(buf); + data(place).deserialize(buf); } static DataTypePtr createResultType() @@ -168,8 +168,8 @@ public: IColumn & to, Arena *) const override { - Float64 k = this->data(place).getK(); - Float64 b = this->data(place).getB(k); + Float64 k = data(place).getK(); + Float64 b = data(place).getB(k); auto & col_tuple = assert_cast(to); auto & col_k = assert_cast &>(col_tuple.getColumn(0)); diff --git a/src/AggregateFunctions/AggregateFunctionSingleValueOrNull.cpp b/src/AggregateFunctions/AggregateFunctionSingleValueOrNull.cpp index b14af34c5fc..0625e37d1b0 100644 --- a/src/AggregateFunctions/AggregateFunctionSingleValueOrNull.cpp +++ b/src/AggregateFunctions/AggregateFunctionSingleValueOrNull.cpp @@ -120,7 +120,7 @@ public: void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { - this->data(place).add(*columns[0], row_num, arena); + data(place).add(*columns[0], row_num, arena); } void addBatchSinglePlace( @@ -131,7 +131,7 @@ public: Arena * arena, ssize_t if_argument_pos) const override { - if (this->data(place).isNull()) + if (data(place).isNull()) return; IAggregateFunctionDataHelper::addBatchSinglePlace( row_begin, row_end, place, columns, arena, if_argument_pos); @@ -146,7 +146,7 @@ public: Arena * arena, ssize_t if_argument_pos) const override { - if (this->data(place).isNull()) + if (data(place).isNull()) return; IAggregateFunctionDataHelper::addBatchSinglePlaceNotNull( row_begin, row_end, place, columns, null_map, arena, if_argument_pos); @@ -154,29 +154,29 @@ public: void addManyDefaults(AggregateDataPtr __restrict place, const IColumn ** columns, size_t, Arena * arena) const override { - this->data(place).add(*columns[0], 0, arena); + data(place).add(*columns[0], 0, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - this->data(place).add(this->data(rhs), arena); + data(place).add(data(rhs), arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf, *serialization); + data(place).write(buf, *serialization); } void deserialize(AggregateDataPtr place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, *serialization, arena); + data(place).read(buf, *serialization, arena); } bool allocatesMemoryInArena() const override { return singleValueTypeAllocatesMemoryInArena(value_type_index); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - this->data(place).insertResultInto(to); + data(place).insertResultInto(to); } }; diff --git a/src/AggregateFunctions/AggregateFunctionStatistics.cpp b/src/AggregateFunctions/AggregateFunctionStatistics.cpp index e9d9b7409ca..15fede94fe7 100644 --- a/src/AggregateFunctions/AggregateFunctionStatistics.cpp +++ b/src/AggregateFunctions/AggregateFunctionStatistics.cpp @@ -150,13 +150,13 @@ private: Float64 getResult(ConstAggregateDataPtr __restrict place) const { - const auto & data = this->data(place); + const auto & dt = data(place); switch (kind) { - case VarKind::varSampStable: return getVarSamp(data.m2, data.count); - case VarKind::stddevSampStable: return getStddevSamp(data.m2, data.count); - case VarKind::varPopStable: return getVarPop(data.m2, data.count); - case VarKind::stddevPopStable: return getStddevPop(data.m2, data.count); + case VarKind::varSampStable: return getVarSamp(dt.m2, dt.count); + case VarKind::stddevSampStable: return getStddevSamp(dt.m2, dt.count); + case VarKind::varPopStable: return getVarPop(dt.m2, dt.count); + case VarKind::stddevPopStable: return getStddevPop(dt.m2, dt.count); } } @@ -182,22 +182,22 @@ public: void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override { - this->data(place).update(*columns[0], row_num); + data(place).update(*columns[0], row_num); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override { - this->data(place).mergeWith(this->data(rhs)); + data(place).mergeWith(data(rhs)); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).serialize(buf); + data(place).serialize(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override { - this->data(place).deserialize(buf); + data(place).deserialize(buf); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h index 306e293cae7..9d13b77664d 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h @@ -491,7 +491,7 @@ public: std::vector nullable_filters; const IColumn * nested_columns[number_of_arguments]; - std::unique_ptr final_flags = nullptr; + std::unique_ptr final_flags; const UInt8 * final_flags_ptr = nullptr; if (if_argument_pos >= 0) diff --git a/src/AggregateFunctions/ReservoirSampler.h b/src/AggregateFunctions/ReservoirSampler.h index 37fc05a2e4c..7b6ef1b2dc0 100644 --- a/src/AggregateFunctions/ReservoirSampler.h +++ b/src/AggregateFunctions/ReservoirSampler.h @@ -258,10 +258,10 @@ private: chassert(limit > 0); /// With a large number of values, we will generate random numbers several times slower. - if (limit <= static_cast(rng.max())) + if (limit <= static_cast(pcg32_fast::max())) return rng() % limit; else - return (static_cast(rng()) * (static_cast(rng.max()) + 1ULL) + static_cast(rng())) % limit; + return (static_cast(rng()) * (static_cast(pcg32_fast::max()) + 1ULL) + static_cast(rng())) % limit; } void sortIfNeeded() diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f5474ddb662..fffb8f7f281 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1940,8 +1940,7 @@ std::vector QueryAnalyzer::collectIdentifierTypoHints(const Identifier & for (const auto & valid_identifier : valid_identifiers) prompting_strings.push_back(valid_identifier.getFullName()); - NamePrompter<1> prompter; - return prompter.getHints(unresolved_identifier.getFullName(), prompting_strings); + return NamePrompter<1>::getHints(unresolved_identifier.getFullName(), prompting_strings); } /** Wrap expression node in tuple element function calls for nested paths. diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index b9a173fd92c..3c08ebbf8b4 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -346,7 +346,7 @@ ColumnPtr ColumnSparse::filter(const Filter & filt, ssize_t) const } auto res_values = values->filter(values_filter, values_result_size_hint); - return this->create(res_values, std::move(res_offsets), res_offset); + return create(res_values, std::move(res_offsets), res_offset); } void ColumnSparse::expand(const Filter & mask, bool inverted) diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index 0b9be18c84e..ab54b180fbf 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -671,7 +671,7 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) ReadableSize(rss), ReadableSize(difference)); - total_memory_tracker.setRSS(rss, free_memory_in_allocator_arenas); + MemoryTracker::setRSS(rss, free_memory_in_allocator_arenas); } } diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index cd2505df7f3..1c011880f8f 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -203,7 +203,7 @@ public: if (total_connections_in_group >= limits.warning_limit && total_connections_in_group >= mute_warning_until) { LOG_WARNING(log, "Too many active sessions in group {}, count {}, warning limit {}", type, total_connections_in_group, limits.warning_limit); - mute_warning_until = roundUp(total_connections_in_group, limits.warning_step); + mute_warning_until = roundUp(total_connections_in_group, HTTPConnectionPools::Limits::warning_step); } } diff --git a/src/Common/HashTable/HashMap.h b/src/Common/HashTable/HashMap.h index 5f4cb396822..dc601bf1319 100644 --- a/src/Common/HashTable/HashMap.h +++ b/src/Common/HashTable/HashMap.h @@ -207,7 +207,7 @@ public: void ALWAYS_INLINE mergeToViaEmplace(Self & that, Func && func) { DB::PrefetchingHelper prefetching; - size_t prefetch_look_ahead = prefetching.getInitialLookAheadValue(); + size_t prefetch_look_ahead = DB::PrefetchingHelper::getInitialLookAheadValue(); size_t i = 0; auto prefetch_it = advanceIterator(this->begin(), prefetch_look_ahead); @@ -216,10 +216,10 @@ public: { if constexpr (prefetch) { - if (i == prefetching.iterationsToMeasure()) + if (i == DB::PrefetchingHelper::iterationsToMeasure()) { prefetch_look_ahead = prefetching.calcPrefetchLookAhead(); - prefetch_it = advanceIterator(prefetch_it, prefetch_look_ahead - prefetching.getInitialLookAheadValue()); + prefetch_it = advanceIterator(prefetch_it, prefetch_look_ahead - DB::PrefetchingHelper::getInitialLookAheadValue()); } if (prefetch_it != end) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 436b85ff30b..891850ccb79 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -366,7 +366,7 @@ String demangleAndCollapseNames(std::optional file, const char if (file.has_value()) { std::string_view file_copy = file.value(); - if (auto trim_pos = file_copy.find_last_of('/'); trim_pos != file_copy.npos) + if (auto trim_pos = file_copy.find_last_of('/'); trim_pos != std::string_view::npos) file_copy.remove_suffix(file_copy.size() - trim_pos); if (file_copy.ends_with("functional")) return "?"; diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index cf50d305e95..8719a9e093a 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -96,7 +96,7 @@ ThreadStatus::ThreadStatus(bool check_current_thread_on_destruction_) stack_t altstack_description{}; altstack_description.ss_sp = alt_stack.getData(); altstack_description.ss_flags = 0; - altstack_description.ss_size = alt_stack.getSize(); + altstack_description.ss_size = ThreadStack::getSize(); if (0 != sigaltstack(&altstack_description, nullptr)) { diff --git a/src/Compression/CachedCompressedReadBuffer.cpp b/src/Compression/CachedCompressedReadBuffer.cpp index 0febfca75cc..3476f436eeb 100644 --- a/src/Compression/CachedCompressedReadBuffer.cpp +++ b/src/Compression/CachedCompressedReadBuffer.cpp @@ -38,7 +38,7 @@ void CachedCompressedReadBuffer::prefetch(Priority priority) bool CachedCompressedReadBuffer::nextImpl() { /// Let's check for the presence of a decompressed block in the cache, grab the ownership of this block, if it exists. - UInt128 key = cache->hash(path, file_pos); + UInt128 key = UncompressedCache::hash(path, file_pos); owned_cell = cache->getOrSet(key, [&]() { diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index d8acfb5fa01..674e9afa8ac 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -68,7 +68,7 @@ std::pair createTableFromAST( ast_create_query.setDatabase(database_name); if (ast_create_query.select && ast_create_query.isView()) - ApplyWithSubqueryVisitor().visit(*ast_create_query.select); + ApplyWithSubqueryVisitor::visit(*ast_create_query.select); if (ast_create_query.as_table_function) { diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 95bdcfc7dce..90f777d7d1d 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -171,7 +171,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables auto ast = parseQueryFromMetadata(log, getContext(), full_path.string(), /*throw_on_error*/ true, /*remove_empty*/ false); if (ast) { - FunctionNameNormalizer().visit(ast.get()); + FunctionNameNormalizer::visit(ast.get()); auto * create_query = ast->as(); /// NOTE No concurrent writes are possible during database loading create_query->setDatabase(TSA_SUPPRESS_WARNING_FOR_READ(database_name)); diff --git a/src/Databases/TablesDependencyGraph.cpp b/src/Databases/TablesDependencyGraph.cpp index 4b05f19fe91..d227a3ac76b 100644 --- a/src/Databases/TablesDependencyGraph.cpp +++ b/src/Databases/TablesDependencyGraph.cpp @@ -448,7 +448,7 @@ std::vector TablesDependencyGraph::getTables() const void TablesDependencyGraph::mergeWith(const TablesDependencyGraph & other) { for (const auto & other_node : other.nodes) - addDependencies(other_node->storage_id, other.getDependencies(*other_node)); + addDependencies(other_node->storage_id, TablesDependencyGraph::getDependencies(*other_node)); } diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index fc58ff525bd..e3b1e8a84e2 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -412,8 +412,7 @@ void FlatDictionary::blockToAttributes(const Block & block) { const auto keys_column = block.safeGetByPosition(0).column; - DictionaryKeysArenaHolder arena_holder; - DictionaryKeysExtractor keys_extractor({ keys_column }, arena_holder.getComplexKeyArena()); + DictionaryKeysExtractor keys_extractor({ keys_column }, DictionaryKeysArenaHolder::getComplexKeyArena()); size_t keys_size = keys_extractor.getKeysSize(); static constexpr size_t key_offset = 1; diff --git a/src/Dictionaries/PolygonDictionaryUtils.h b/src/Dictionaries/PolygonDictionaryUtils.h index 0acf0d23e5e..0fd1fead456 100644 --- a/src/Dictionaries/PolygonDictionaryUtils.h +++ b/src/Dictionaries/PolygonDictionaryUtils.h @@ -214,7 +214,7 @@ public: static constexpr Coord kEps = 1e-4f; private: - std::unique_ptr> root = nullptr; + std::unique_ptr> root; Coord min_x = 0, min_y = 0; Coord max_x = 0, max_y = 0; const size_t k_min_intersections; diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 1a8d46668e0..33f7ca1ec19 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -581,7 +581,7 @@ try auto disk_ptr = std::static_pointer_cast(shared_from_this()); auto tmp_file = std::make_unique(disk_ptr); auto buf = std::make_unique(std::move(tmp_file)); - buf->write(data.data, data.PAGE_SIZE_IN_BYTES); + buf->write(data.data, DiskWriteCheckData::PAGE_SIZE_IN_BYTES); buf->finalize(); buf->sync(); } diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index d3317e2dfcf..4460a8bd7bd 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -1139,7 +1139,7 @@ private: getName()); auto dictionary = helper.getDictionary(arguments[0].column); - const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); + const auto & hierarchical_attribute = FunctionDictHelper::getDictionaryHierarchicalAttribute(dictionary); return std::make_shared(removeNullable(hierarchical_attribute.type)); } @@ -1150,7 +1150,7 @@ private: return result_type->createColumn(); auto dictionary = helper.getDictionary(arguments[0].column); - const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); + const auto & hierarchical_attribute = FunctionDictHelper::getDictionaryHierarchicalAttribute(dictionary); auto key_column = ColumnWithTypeAndName{arguments[1].column, arguments[1].type, arguments[1].name}; auto key_column_casted = castColumnAccurate(key_column, removeNullable(hierarchical_attribute.type)); @@ -1205,7 +1205,7 @@ private: return result_type->createColumn(); auto dictionary = helper.getDictionary(arguments[0].column); - const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); + const auto & hierarchical_attribute = FunctionDictHelper::getDictionaryHierarchicalAttribute(dictionary); auto key_column = ColumnWithTypeAndName{arguments[1].column->convertToFullColumnIfConst(), arguments[1].type, arguments[2].name}; auto in_key_column = ColumnWithTypeAndName{arguments[2].column->convertToFullColumnIfConst(), arguments[2].type, arguments[2].name}; diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp index 4f47a46b10d..f251d11789f 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp @@ -23,7 +23,7 @@ ASTPtr normalizeCreateFunctionQuery(const IAST & create_function_query) auto & res = typeid_cast(*ptr); res.if_not_exists = false; res.or_replace = false; - FunctionNameNormalizer().visit(res.function_core.get()); + FunctionNameNormalizer::visit(res.function_core.get()); return ptr; } diff --git a/src/IO/Archives/LibArchiveWriter.h b/src/IO/Archives/LibArchiveWriter.h index f54a8ce2367..da566c82ff6 100644 --- a/src/IO/Archives/LibArchiveWriter.h +++ b/src/IO/Archives/LibArchiveWriter.h @@ -68,7 +68,7 @@ private: void startWritingFile(); void endWritingFile(); - std::unique_ptr stream_info TSA_GUARDED_BY(mutex) = nullptr; + std::unique_ptr stream_info TSA_GUARDED_BY(mutex); bool is_writing_file TSA_GUARDED_BY(mutex) = false; bool finalized TSA_GUARDED_BY(mutex) = false; mutable std::mutex mutex; diff --git a/src/IO/MMapReadBufferFromFileWithCache.cpp b/src/IO/MMapReadBufferFromFileWithCache.cpp index d53f3bc325d..68c0c7227ca 100644 --- a/src/IO/MMapReadBufferFromFileWithCache.cpp +++ b/src/IO/MMapReadBufferFromFileWithCache.cpp @@ -26,7 +26,7 @@ void MMapReadBufferFromFileWithCache::init() MMapReadBufferFromFileWithCache::MMapReadBufferFromFileWithCache( MMappedFileCache & cache, const std::string & file_name, size_t offset, size_t length) { - mapped = cache.getOrSet(cache.hash(file_name, offset, length), [&] + mapped = cache.getOrSet(MMappedFileCache::hash(file_name, offset, length), [&] { return std::make_shared(file_name, offset, length); }); diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index fc105539061..a9c861be13c 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -822,7 +822,7 @@ inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, cons return false; /// When the parameter is out of rule or out of range, Date32 uses 1925-01-01 as the default value (-DateLUT::instance().getDayNumOffsetEpoch(), -16436) and Date uses 1970-01-01. - date = date_lut.makeDayNum(local_date.year(), local_date.month(), local_date.day(), -static_cast(date_lut.getDayNumOffsetEpoch())); + date = date_lut.makeDayNum(local_date.year(), local_date.month(), local_date.day(), -static_cast(DateLUTImpl::getDayNumOffsetEpoch())); return ReturnType(true); } @@ -1880,10 +1880,10 @@ struct PcgDeserializer assertChar(' ', buf); readText(state, buf); - if (multiplier != rng.multiplier()) - throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect multiplier in pcg32: expected {}, got {}", rng.multiplier(), multiplier); - if (increment != rng.increment()) - throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect increment in pcg32: expected {}, got {}", rng.increment(), increment); + if (multiplier != pcg32_fast::multiplier()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect multiplier in pcg32: expected {}, got {}", pcg32_fast::multiplier(), multiplier); + if (increment != pcg32_fast::increment()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect increment in pcg32: expected {}, got {}", pcg32_fast::increment(), increment); rng.state_ = state; } diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index b42b4e6e978..a30e2feb439 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -1390,9 +1390,9 @@ struct PcgSerializer { static void serializePcg32(const pcg32_fast & rng, WriteBuffer & buf) { - writeText(rng.multiplier(), buf); + writeText(pcg32_fast::multiplier(), buf); writeChar(' ', buf); - writeText(rng.increment(), buf); + writeText(pcg32_fast::increment(), buf); writeChar(' ', buf); writeText(rng.state_, buf); } diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index c05d1b8f979..6b9ca34c2d7 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -294,7 +294,7 @@ void AsynchronousInsertQueue::preprocessInsertQuery(const ASTPtr & query, const InterpreterInsertQuery interpreter(query, query_context, query_context->getSettingsRef().insert_allow_materialized_columns); auto table = interpreter.getTable(insert_query); - auto sample_block = interpreter.getSampleBlock(insert_query, table, table->getInMemoryMetadataPtr(), query_context); + auto sample_block = InterpreterInsertQuery::getSampleBlock(insert_query, table, table->getInMemoryMetadataPtr(), query_context); if (!FormatFactory::instance().isInputFormat(insert_query.format)) throw Exception(ErrorCodes::UNKNOWN_FORMAT, "Unknown input format {}", insert_query.format); diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 0cf138c14f6..a70ff3c6c53 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -112,7 +112,7 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr auto hash = subquery.getTreeHash(/*ignore_aliases=*/ true); const auto scalar_query_hash_str = toString(hash); - std::unique_ptr interpreter = nullptr; + std::unique_ptr interpreter; bool hit = false; bool is_local = false; diff --git a/src/Interpreters/InterpreterCreateFunctionQuery.cpp b/src/Interpreters/InterpreterCreateFunctionQuery.cpp index 18e9ba4a64b..3f4a03c3497 100644 --- a/src/Interpreters/InterpreterCreateFunctionQuery.cpp +++ b/src/Interpreters/InterpreterCreateFunctionQuery.cpp @@ -21,7 +21,7 @@ namespace ErrorCodes BlockIO InterpreterCreateFunctionQuery::execute() { - FunctionNameNormalizer().visit(query_ptr.get()); + FunctionNameNormalizer::visit(query_ptr.get()); const auto updated_query_ptr = removeOnClusterClauseIfNeeded(query_ptr, getContext()); ASTCreateFunctionQuery & create_function_query = updated_query_ptr->as(); diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 07d23be78a7..ee774994145 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -35,7 +35,7 @@ InterpreterDeleteQuery::InterpreterDeleteQuery(const ASTPtr & query_ptr_, Contex BlockIO InterpreterDeleteQuery::execute() { - FunctionNameNormalizer().visit(query_ptr.get()); + FunctionNameNormalizer::visit(query_ptr.get()); const ASTDeleteQuery & delete_query = query_ptr->as(); auto table_id = getContext()->resolveStorageID(delete_query, Context::ResolveOrdinary); diff --git a/src/Interpreters/InterpreterDropFunctionQuery.cpp b/src/Interpreters/InterpreterDropFunctionQuery.cpp index 2661fd9058c..7a273d4969b 100644 --- a/src/Interpreters/InterpreterDropFunctionQuery.cpp +++ b/src/Interpreters/InterpreterDropFunctionQuery.cpp @@ -21,7 +21,7 @@ namespace ErrorCodes BlockIO InterpreterDropFunctionQuery::execute() { - FunctionNameNormalizer().visit(query_ptr.get()); + FunctionNameNormalizer::visit(query_ptr.get()); const auto updated_query_ptr = removeOnClusterClauseIfNeeded(query_ptr, getContext()); ASTDropFunctionQuery & drop_function_query = updated_query_ptr->as(); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c47e3bdc49f..75baefeffba 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -412,8 +412,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (!options.is_subquery) { if (context->getSettingsRef().enable_global_with_statement) - ApplyWithAliasVisitor().visit(query_ptr); - ApplyWithSubqueryVisitor().visit(query_ptr); + ApplyWithAliasVisitor::visit(query_ptr); + ApplyWithSubqueryVisitor::visit(query_ptr); } query_info.query = query_ptr->clone(); @@ -609,7 +609,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (view) { query_info.is_parameterized_view = view->isParameterizedView(); - view->replaceWithSubquery(getSelectQuery(), view_table, metadata_snapshot, view->isParameterizedView()); + StorageView::replaceWithSubquery(getSelectQuery(), view_table, metadata_snapshot, view->isParameterizedView()); } syntax_analyzer_result = TreeRewriter(context).analyzeSelect( @@ -629,7 +629,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (view) { /// Restore original view name. Save rewritten subquery for future usage in StorageView. - query_info.view_query = view->restoreViewName(getSelectQuery(), view_table); + query_info.view_query = StorageView::restoreViewName(getSelectQuery(), view_table); view = nullptr; } @@ -2434,7 +2434,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc agg_count.create(place); SCOPE_EXIT_MEMORY_SAFE(agg_count.destroy(place)); - agg_count.set(place, *num_rows); + AggregateFunctionCount::set(place, *num_rows); auto column = ColumnAggregateFunction::create(func); column->insertFrom(place); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 5588fc55a64..914b3c3037d 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1587,7 +1587,7 @@ void TreeRewriter::normalize( /// already normalized on initiator node, or not normalized and should remain unnormalized for /// compatibility. if (context_->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && settings.normalize_function_names) - FunctionNameNormalizer().visit(query.get()); + FunctionNameNormalizer::visit(query.get()); if (settings.optimize_move_to_prewhere) { diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index b5c3e00e299..4e1a2bcf5ee 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -73,7 +73,7 @@ std::optional evaluateConstantExpressionImpl(c /// already normalized on initiator node, or not normalized and should remain unnormalized for /// compatibility. if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && context->getSettingsRef().normalize_function_names) - FunctionNameNormalizer().visit(ast.get()); + FunctionNameNormalizer::visit(ast.get()); auto syntax_result = TreeRewriter(context, no_throw).analyze(ast, source_columns); if (!syntax_result) diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 326dd683343..656b6cdaa6e 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -169,7 +169,7 @@ public: { const auto & in_first_argument_node = function_node.getArguments().getNodes().at(0); const auto & in_second_argument_node = function_node.getArguments().getNodes().at(1); - in_function_second_argument_node_name = planner_context.createSetKey(in_first_argument_node->getResultType(), in_second_argument_node); + in_function_second_argument_node_name = PlannerContext::createSetKey(in_first_argument_node->getResultType(), in_second_argument_node); } WriteBufferFromOwnString buffer; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index e9c3795176a..adab31adb40 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -305,7 +305,7 @@ bool applyTrivialCountIfPossible( AggregateDataPtr place = state.data(); agg_count.create(place); SCOPE_EXIT_MEMORY_SAFE(agg_count.destroy(place)); - agg_count.set(place, num_rows.value()); + AggregateFunctionCount::set(place, num_rows.value()); auto column = ColumnAggregateFunction::create(function_node.getAggregateFunction()); column->insertFrom(place); diff --git a/src/Processors/Executors/ExecutorTasks.cpp b/src/Processors/Executors/ExecutorTasks.cpp index ec1fc539884..1039cf0e97a 100644 --- a/src/Processors/Executors/ExecutorTasks.cpp +++ b/src/Processors/Executors/ExecutorTasks.cpp @@ -121,7 +121,7 @@ void ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThrea /// Take local task from queue if has one. if (!queue.empty() && !context.hasAsyncTasks() - && context.num_scheduled_local_tasks < context.max_scheduled_local_tasks) + && context.num_scheduled_local_tasks < ExecutionThreadContext::max_scheduled_local_tasks) { ++context.num_scheduled_local_tasks; context.setTask(queue.front()); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index f4607cad040..bee42c3ddde 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1408,8 +1408,8 @@ static void buildIndexes( if (metadata_snapshot->hasPartitionKey()) { const auto & partition_key = metadata_snapshot->getPartitionKey(); - auto minmax_columns_names = data.getMinMaxColumnsNames(partition_key); - auto minmax_expression_actions = data.getMinMaxExpr(partition_key, ExpressionActionsSettings::fromContext(context)); + auto minmax_columns_names = MergeTreeData::getMinMaxColumnsNames(partition_key); + auto minmax_expression_actions = MergeTreeData::getMinMaxExpr(partition_key, ExpressionActionsSettings::fromContext(context)); indexes->minmax_idx_condition.emplace(filter_actions_dag, context, minmax_columns_names, minmax_expression_actions); indexes->partition_pruner.emplace(metadata_snapshot, filter_actions_dag, context, false /* strict */); diff --git a/src/Server/HTTP/ReadHeaders.cpp b/src/Server/HTTP/ReadHeaders.cpp index b7057501064..d6c7b8ddc0f 100644 --- a/src/Server/HTTP/ReadHeaders.cpp +++ b/src/Server/HTTP/ReadHeaders.cpp @@ -77,7 +77,7 @@ void readHeaders( skipToNextLineOrEOF(in); Poco::trimRightInPlace(value); - headers.add(name, headers.decodeWord(value)); + headers.add(name, Poco::Net::MessageHeader::decodeWord(value)); ++fields; } } diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index eae5e1a8a47..281fc72dfc4 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1143,7 +1143,7 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, ContextPtr context { auto minmax_columns = metadata_copy.getColumnsRequiredForPartitionKey(); auto partition_key = metadata_copy.partition_key.expression_list_ast->clone(); - FunctionNameNormalizer().visit(partition_key.get()); + FunctionNameNormalizer::visit(partition_key.get()); auto primary_key_asts = metadata_copy.primary_key.expression_list_ast->children; metadata_copy.minmax_count_projection.emplace(ProjectionDescription::getMinMaxCountProjection( metadata_copy.columns, partition_key, minmax_columns, primary_key_asts, context)); diff --git a/src/Storages/FileLog/StorageFileLog.h b/src/Storages/FileLog/StorageFileLog.h index 91d58540c94..0434213c558 100644 --- a/src/Storages/FileLog/StorageFileLog.h +++ b/src/Storages/FileLog/StorageFileLog.h @@ -177,7 +177,7 @@ private: }; std::shared_ptr task; - std::unique_ptr directory_watch = nullptr; + std::unique_ptr directory_watch; void loadFiles(); diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index d63b40e2b11..2a697fa5654 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -172,7 +172,7 @@ KeyDescription KeyDescription::parse(const String & str, const ColumnsDescriptio ParserExpression parser; ASTPtr ast = parseQuery(parser, "(" + str + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); - FunctionNameNormalizer().visit(ast.get()); + FunctionNameNormalizer::visit(ast.get()); return getKeyFromAST(ast, columns, context); } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 570175f6614..8da46b39801 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -79,8 +79,8 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Par auto metadata_snapshot = data.getInMemoryMetadataPtr(); const auto & partition_key = metadata_snapshot->getPartitionKey(); - auto minmax_column_names = data.getMinMaxColumnsNames(partition_key); - auto minmax_column_types = data.getMinMaxColumnsTypes(partition_key); + auto minmax_column_names = MergeTreeData::getMinMaxColumnsNames(partition_key); + auto minmax_column_types = MergeTreeData::getMinMaxColumnsTypes(partition_key); size_t minmax_idx_size = minmax_column_types.size(); hyperrectangle.reserve(minmax_idx_size); @@ -112,8 +112,8 @@ IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::s auto metadata_snapshot = data.getInMemoryMetadataPtr(); const auto & partition_key = metadata_snapshot->getPartitionKey(); - auto minmax_column_names = data.getMinMaxColumnsNames(partition_key); - auto minmax_column_types = data.getMinMaxColumnsTypes(partition_key); + auto minmax_column_names = MergeTreeData::getMinMaxColumnsNames(partition_key); + auto minmax_column_types = MergeTreeData::getMinMaxColumnsTypes(partition_key); return store(minmax_column_names, minmax_column_types, part_storage, out_checksums); } @@ -204,7 +204,7 @@ void IMergeTreeDataPart::MinMaxIndex::appendFiles(const MergeTreeData & data, St { auto metadata_snapshot = data.getInMemoryMetadataPtr(); const auto & partition_key = metadata_snapshot->getPartitionKey(); - auto minmax_column_names = data.getMinMaxColumnsNames(partition_key); + auto minmax_column_names = MergeTreeData::getMinMaxColumnsNames(partition_key); size_t minmax_idx_size = minmax_column_names.size(); for (size_t i = 0; i < minmax_idx_size; ++i) { @@ -1213,7 +1213,7 @@ void IMergeTreeDataPart::appendFilesOfPartitionAndMinMaxIndex(Strings & files) c return; if (!parent_part) - partition.appendFiles(storage, files); + MergeTreePartition::appendFiles(storage, files); if (!parent_part) minmax_idx->appendFiles(storage, files); @@ -2061,7 +2061,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const if (!isEmpty() && !parent_part) { - for (const String & col_name : storage.getMinMaxColumnsNames(partition_key)) + for (const String & col_name : MergeTreeData::getMinMaxColumnsNames(partition_key)) { if (!checksums.files.contains("minmax_" + escapeForFileName(col_name) + ".idx")) throw Exception(ErrorCodes::NO_FILE_IN_DATA_PART, "No minmax idx file checksum for column {}", col_name); @@ -2101,7 +2101,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const if (!parent_part) { - for (const String & col_name : storage.getMinMaxColumnsNames(partition_key)) + for (const String & col_name : MergeTreeData::getMinMaxColumnsNames(partition_key)) check_file_not_empty("minmax_" + escapeForFileName(col_name) + ".idx"); } } diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 2d57ea40c9c..a779311c22b 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1964,8 +1964,8 @@ KeyCondition::Description KeyCondition::getDescription() const /// This means that logical NOT is applied to leaf. bool negate = false; - std::unique_ptr left = nullptr; - std::unique_ptr right = nullptr; + std::unique_ptr left; + std::unique_ptr right; }; /// The algorithm is the same as in KeyCondition::checkInHyperrectangle diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 8faed72b198..e5ace0e5969 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5362,7 +5362,7 @@ void MergeTreeData::restoreDataFromBackup(RestorerFromBackup & restorer, const S return; if (!restorer.isNonEmptyTableAllowed() && getTotalActiveSizeInBytes() && backup->hasFiles(data_path_in_backup)) - restorer.throwTableIsNotEmpty(getStorageID()); + RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); restorePartsFromBackup(restorer, data_path_in_backup, partitions); } @@ -6687,7 +6687,7 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( auto * place = arena.alignedAlloc(size_of_state, align_of_state); func->create(place); if (const AggregateFunctionCount * agg_count = typeid_cast(func.get())) - agg_count->set(place, value.get()); + AggregateFunctionCount::set(place, value.get()); else { auto value_column = func->getArgumentTypes().front()->createColumnConst(1, value)->convertToFullColumnIfConst(); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 6471f510291..bcc936c5739 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -513,11 +513,11 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( { chassert(minmax_idx_condition && partition_pruner); const auto & partition_key = metadata_snapshot->getPartitionKey(); - minmax_columns_types = data.getMinMaxColumnsTypes(partition_key); + minmax_columns_types = MergeTreeData::getMinMaxColumnsTypes(partition_key); if (settings.force_index_by_date && (minmax_idx_condition->alwaysUnknownOrTrue() && partition_pruner->isUseless())) { - auto minmax_columns_names = data.getMinMaxColumnsNames(partition_key); + auto minmax_columns_names = MergeTreeData::getMinMaxColumnsNames(partition_key); throw Exception(ErrorCodes::INDEX_NOT_USED, "Neither MinMax index by columns ({}) nor partition expr is used and setting 'force_index_by_date' is set", fmt::join(minmax_columns_names, ", ")); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index cadd94867ec..64d4b1fd7ff 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -426,7 +426,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( column.type = block.getByName(column.name).type; auto minmax_idx = std::make_shared(); - minmax_idx->update(block, data.getMinMaxColumnsNames(metadata_snapshot->getPartitionKey())); + minmax_idx->update(block, MergeTreeData::getMinMaxColumnsNames(metadata_snapshot->getPartitionKey())); MergeTreePartition partition(block_with_partition.partition); @@ -656,7 +656,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( /// Size of part would not be greater than block.bytes() + epsilon size_t expected_size = block.bytes(); // just check if there is enough space on parent volume - data.reserveSpace(expected_size, parent_part->getDataPartStorage()); + MergeTreeData::reserveSpace(expected_size, parent_part->getDataPartStorage()); part_type = data.choosePartFormatOnDisk(expected_size, block.rows()).part_type; auto new_data_part = parent_part->getProjectionPartBuilder(part_name, is_temp).withPartType(part_type).build(); diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 3ead766cba9..82c015f33ed 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -195,7 +195,7 @@ void MergeTreeSink::finishDelayedChunk() { auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, partition.elapsed_ns, counters_snapshot)); - storage.incrementInsertedPartsProfileEvent(part->getType()); + StorageMergeTree::incrementInsertedPartsProfileEvent(part->getType()); /// Initiate async merge - it will be done if it's good time for merge and if there are space in 'background_pool'. storage.background_operations_assignee.trigger(); diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 7536eb45903..3415b08cebb 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -116,7 +116,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() /// Once we mutate part, we must reserve space on the same disk, because mutations can possibly create hardlinks. /// Can throw an exception. - reserved_space = storage.reserveSpace(estimated_space_for_result, source_part->getDataPartStorage()); + reserved_space = StorageReplicatedMergeTree::reserveSpace(estimated_space_for_result, source_part->getDataPartStorage()); future_mutated_part->updatePath(storage, reserved_space.get()); table_lock_holder = storage.lockForShare( diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 90e1cb0606e..a971c4fda1c 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -980,13 +980,13 @@ struct MutationContext QueryPipelineBuilder mutating_pipeline_builder; QueryPipeline mutating_pipeline; // in - std::unique_ptr mutating_executor{nullptr}; + std::unique_ptr mutating_executor; ProgressCallback progress_callback; Block updated_header; std::unique_ptr interpreter; - UInt64 watch_prev_elapsed{0}; - std::unique_ptr stage_progress{nullptr}; + UInt64 watch_prev_elapsed = 0; + std::unique_ptr stage_progress; MutationCommands commands_for_part; MutationCommands for_interpreter; @@ -998,12 +998,12 @@ struct MutationContext NameSet materialized_statistics; MergeTreeData::MutableDataPartPtr new_data_part; - IMergedBlockOutputStreamPtr out{nullptr}; + IMergedBlockOutputStreamPtr out; String mrk_extension; std::vector projections_to_build; - IMergeTreeDataPart::MinMaxIndexPtr minmax_idx{nullptr}; + IMergeTreeDataPart::MinMaxIndexPtr minmax_idx; std::set indices_to_recalc; std::set stats_to_recalc; @@ -1283,7 +1283,7 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections() if (MutationHelpers::checkOperationIsNotCanceled(*ctx->merges_blocker, ctx->mutate_entry) && ctx->mutating_executor->pull(cur_block)) { if (ctx->minmax_idx) - ctx->minmax_idx->update(cur_block, ctx->data->getMinMaxColumnsNames(ctx->metadata_snapshot->getPartitionKey())); + ctx->minmax_idx->update(cur_block, MergeTreeData::getMinMaxColumnsNames(ctx->metadata_snapshot->getPartitionKey())); ctx->out->write(cur_block); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 7fcf6b971bb..e1f28c2a951 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -289,7 +289,7 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk chunk) throw Exception(ErrorCodes::LOGICAL_ERROR, "No chunk info for async inserts"); } - auto part_blocks = storage.writer.splitBlockIntoParts(std::move(block), max_parts_per_block, metadata_snapshot, context, async_insert_info); + auto part_blocks = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), max_parts_per_block, metadata_snapshot, context, async_insert_info); using DelayedPartition = typename ReplicatedMergeTreeSinkImpl::DelayedChunk::Partition; using DelayedPartitions = std::vector; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 0ca7a4d74d9..268f2d66c96 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -33,7 +33,7 @@ static String formattedASTNormalized(const ASTPtr & ast) if (!ast) return ""; auto ast_normalized = ast->clone(); - FunctionNameNormalizer().visit(ast_normalized.get()); + FunctionNameNormalizer::visit(ast_normalized.get()); WriteBufferFromOwnString buf; formatAST(*ast_normalized, buf, false, true); return buf.str(); diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 9a3c17923d8..d552a4b6fa5 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -585,7 +585,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) auto minmax_columns = metadata.getColumnsRequiredForPartitionKey(); auto partition_key = metadata.partition_key.expression_list_ast->clone(); - FunctionNameNormalizer().visit(partition_key.get()); + FunctionNameNormalizer::visit(partition_key.get()); auto primary_key_asts = metadata.primary_key.expression_list_ast->children; metadata.minmax_count_projection.emplace(ProjectionDescription::getMinMaxCountProjection( columns, partition_key, minmax_columns, primary_key_asts, context)); @@ -694,7 +694,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) auto minmax_columns = metadata.getColumnsRequiredForPartitionKey(); auto partition_key = metadata.partition_key.expression_list_ast->clone(); - FunctionNameNormalizer().visit(partition_key.get()); + FunctionNameNormalizer::visit(partition_key.get()); auto primary_key_asts = metadata.primary_key.expression_list_ast->children; metadata.minmax_count_projection.emplace(ProjectionDescription::getMinMaxCountProjection( columns, partition_key, minmax_columns, primary_key_asts, context)); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 0d220f2fd5d..f747bbf6b28 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -1741,7 +1741,7 @@ public: void initialize() { - std::unique_ptr naked_buffer = nullptr; + std::unique_ptr naked_buffer; if (use_table_fd) { naked_buffer = std::make_unique(table_fd, DBMS_DEFAULT_BUFFER_SIZE); diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 588429284f0..945ee4f369f 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -286,7 +286,7 @@ private: std::unique_ptr reader; std::shared_ptr archive_reader; - std::unique_ptr file_enumerator = nullptr; + std::unique_ptr file_enumerator; ColumnsDescription columns_description; NamesAndTypesList requested_columns; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index aad4fc36a1b..c9f451b6bb1 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -436,7 +436,7 @@ CurrentlyMergingPartsTagger::CurrentlyMergingPartsTagger( /// if we mutate part, than we should reserve space on the same disk, because mutations possible can create hardlinks if (is_mutation) { - reserved_space = storage.tryReserveSpace(total_size, future_part->parts[0]->getDataPartStorage()); + reserved_space = StorageMergeTree::tryReserveSpace(total_size, future_part->parts[0]->getDataPartStorage()); } else { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8ca061db4ec..c4b84a0ae8c 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -10512,7 +10512,7 @@ void StorageReplicatedMergeTree::restoreDataFromBackup(RestorerFromBackup & rest } auto backup = restorer.getBackup(); if (!empty && backup->hasFiles(data_path_in_backup)) - restorer.throwTableIsNotEmpty(getStorageID()); + RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); } restorePartsFromBackup(restorer, data_path_in_backup, partitions); diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 2d3aef312bf..6cda0fca60b 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -207,7 +207,7 @@ public: , list_objects_scheduler(threadPoolCallbackRunner(list_objects_pool, "ListObjects")) , file_progress_callback(file_progress_callback_) { - if (globbed_uri.bucket.find_first_of("*?{") != globbed_uri.bucket.npos) + if (globbed_uri.bucket.find_first_of("*?{") != std::string::npos) throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION, "Expression can not have wildcards inside bucket name"); const String key_prefix = globbed_uri.key.substr(0, globbed_uri.key.find_first_of("*?{")); From 9626506585666a220b5d82fa276fac45e4bf3ccf Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 3 Apr 2024 19:18:31 +0000 Subject: [PATCH 0969/1165] More fixes --- src/Coordination/FourLetterCommand.cpp | 2 +- .../UserDefinedSQLFunctionFactory.cpp | 2 +- src/Functions/decodeHTMLComponent.cpp | 2 +- src/Processors/Formats/Impl/NativeFormat.cpp | 2 +- src/Processors/Merges/Algorithms/Graphite.cpp | 2 +- src/Processors/QueryPlan/JoinStep.cpp | 2 +- src/Storages/Distributed/DistributedSink.cpp | 2 +- src/Storages/MergeTree/KeyCondition.cpp | 4 ++-- src/Storages/MergeTree/MergeFromLogEntryTask.cpp | 2 +- .../MergeTree/MergePlainMergeTreeTask.cpp | 2 +- .../MergeTreeIndexConditionBloomFilter.cpp | 2 +- src/Storages/MergeTree/MergeTreeMarksLoader.cpp | 2 +- src/Storages/MergeTree/MergeTreePartsMover.cpp | 2 +- src/Storages/MergeTree/MergeTreeSettings.cpp | 2 +- src/Storages/MergeTree/MergeTreeSink.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 2 +- .../ReplicatedMergeTreeTableMetadata.cpp | 2 +- src/Storages/StorageFuzzJSON.cpp | 16 ++++++++-------- src/Storages/TTLDescription.cpp | 2 +- 19 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/Coordination/FourLetterCommand.cpp b/src/Coordination/FourLetterCommand.cpp index d7fa5abe742..25254e10441 100644 --- a/src/Coordination/FourLetterCommand.cpp +++ b/src/Coordination/FourLetterCommand.cpp @@ -592,7 +592,7 @@ String RecalculateCommand::run() String CleanResourcesCommand::run() { - keeper_dispatcher.cleanResources(); + KeeperDispatcher::cleanResources(); return "ok"; } diff --git a/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp b/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp index e37e4a23b63..e22cd6d0022 100644 --- a/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp @@ -86,7 +86,7 @@ namespace auto & res = typeid_cast(*ptr); res.if_not_exists = false; res.or_replace = false; - FunctionNameNormalizer().visit(res.function_core.get()); + FunctionNameNormalizer::visit(res.function_core.get()); return ptr; } } diff --git a/src/Functions/decodeHTMLComponent.cpp b/src/Functions/decodeHTMLComponent.cpp index 2cd95127266..cb6ba0b07e2 100644 --- a/src/Functions/decodeHTMLComponent.cpp +++ b/src/Functions/decodeHTMLComponent.cpp @@ -108,7 +108,7 @@ namespace // null terminate the sequence seq.push_back('\0'); // lookup the html sequence in the perfect hashmap. - const auto * res = hash.Lookup(seq.data(), strlen(seq.data())); + const auto * res = HTMLCharacterHash::Lookup(seq.data(), strlen(seq.data())); // reset so that it's reused in the next iteration seq.clear(); if (res) diff --git a/src/Processors/Formats/Impl/NativeFormat.cpp b/src/Processors/Formats/Impl/NativeFormat.cpp index 73ffc02bbc1..a7a49ab6a8c 100644 --- a/src/Processors/Formats/Impl/NativeFormat.cpp +++ b/src/Processors/Formats/Impl/NativeFormat.cpp @@ -82,7 +82,7 @@ public: std::string getContentType() const override { - return writer.getContentType(); + return NativeWriter::getContentType(); } protected: diff --git a/src/Processors/Merges/Algorithms/Graphite.cpp b/src/Processors/Merges/Algorithms/Graphite.cpp index 817961b709c..a75c2b87720 100644 --- a/src/Processors/Merges/Algorithms/Graphite.cpp +++ b/src/Processors/Merges/Algorithms/Graphite.cpp @@ -76,7 +76,7 @@ inline static const Patterns & selectPatternsForMetricType(const Graphite::Param if (params.patterns_typed) { std::string_view path_view = path; - if (path_view.find("?"sv) == path_view.npos) + if (path_view.find("?"sv) == std::string::npos) return params.patterns_plain; else return params.patterns_tagged; diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index 1931b1eb3a1..8fe2515e323 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -31,7 +31,7 @@ std::vector> describeJoinActions(const JoinPtr & join) description.emplace_back("ASOF inequality", toString(table_join.getAsofInequality())); if (!table_join.getClauses().empty()) - description.emplace_back("Clauses", table_join.formatClauses(table_join.getClauses(), true /*short_format*/)); + description.emplace_back("Clauses", TableJoin::formatClauses(table_join.getClauses(), true /*short_format*/)); return description; } diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index ddbcc6d473f..b89a8d7bcfd 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -613,7 +613,7 @@ IColumn::Selector DistributedSink::createSelector(const Block & source_block) co const auto & key_column = current_block_with_sharding_key_expr.getByName(storage.getShardingKeyColumnName()); - return storage.createSelector(cluster, key_column); + return StorageDistributed::createSelector(cluster, key_column); } diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index a779311c22b..2d57ea40c9c 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1964,8 +1964,8 @@ KeyCondition::Description KeyCondition::getDescription() const /// This means that logical NOT is applied to leaf. bool negate = false; - std::unique_ptr left; - std::unique_ptr right; + std::unique_ptr left = nullptr; + std::unique_ptr right = nullptr; }; /// The algorithm is the same as in KeyCondition::checkInHyperrectangle diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 3d1c5db07b5..e8d55f75b08 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -426,7 +426,7 @@ bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrite ProfileEvents::increment(ProfileEvents::ReplicatedPartMerges); write_part_log({}); - storage.incrementMergedPartsProfileEvent(part->getType()); + StorageReplicatedMergeTree::incrementMergedPartsProfileEvent(part->getType()); return true; } diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index c218acce903..866a63911c3 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -149,7 +149,7 @@ void MergePlainMergeTreeTask::finish() ThreadFuzzer::maybeInjectMemoryLimitException(); write_part_log({}); - storage.incrementMergedPartsProfileEvent(new_part->getType()); + StorageMergeTree::incrementMergedPartsProfileEvent(new_part->getType()); transfer_profile_counters_to_initial_query(); if (auto txn_ = txn_holder.getTransaction()) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index f506230b5ea..7ab90dac5b0 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -590,7 +590,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( for (const auto & f : value_field.get()) { - if ((f.isNull() && !is_nullable) || f.isDecimal(f.getType())) + if ((f.isNull() && !is_nullable) || f.isDecimal(f.getType())) /// NOLINT(readability-static-accessed-through-instance) return false; auto converted = convertFieldToType(f, *actual_type); diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 6798f97e494..1e9a320fa95 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -210,7 +210,7 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksSync() if (mark_cache) { - auto key = mark_cache->hash(fs::path(data_part_storage->getFullPath()) / mrk_path); + auto key = MarkCache::hash(fs::path(data_part_storage->getFullPath()) / mrk_path); if (save_marks_in_cache) { auto callback = [this] { return loadMarksImpl(); }; diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index d32bc6d1826..1db70162bff 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -158,7 +158,7 @@ bool MergeTreePartsMover::selectPartsForMove( { auto destination = data->getDestinationForMoveTTL(*ttl_entry); if (destination && !data->isPartInTTLDestination(*ttl_entry, *part)) - reservation = data->tryReserveSpace(part->getBytesOnDisk(), data->getDestinationForMoveTTL(*ttl_entry)); + reservation = MergeTreeData::tryReserveSpace(part->getBytesOnDisk(), data->getDestinationForMoveTTL(*ttl_entry)); } if (reservation) /// Found reservation by TTL rule. diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index b42da22239e..5d6f08d3c53 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -230,7 +230,7 @@ void MergeTreeColumnSettings::validate(const SettingsChanges & changes) "Setting {} is unknown or not supported at column level, supported settings: {}", change.name, fmt::join(allowed_column_level_settings, ", ")); - merge_tree_settings.checkCanSet(change.name, change.value); + MergeTreeSettings::checkCanSet(change.name, change.value); } } diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 82c015f33ed..b7dede3cb00 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -63,7 +63,7 @@ void MergeTreeSink::consume(Chunk chunk) if (!storage_snapshot->object_columns.empty()) convertDynamicColumnsToTuples(block, storage_snapshot); - auto part_blocks = storage.writer.splitBlockIntoParts(std::move(block), max_parts_per_block, metadata_snapshot, context); + auto part_blocks = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), max_parts_per_block, metadata_snapshot, context); using DelayedPartitions = std::vector; DelayedPartitions partitions; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index e1f28c2a951..8913e9f7e27 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -449,7 +449,7 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithF int error = (deduplicate && deduplicated) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, partition.elapsed_ns, counters_snapshot), ExecutionStatus(error)); - storage.incrementInsertedPartsProfileEvent(part->getType()); + StorageReplicatedMergeTree::incrementInsertedPartsProfileEvent(part->getType()); } catch (...) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 268f2d66c96..287a4d20543 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -43,7 +43,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr { if (data.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - auto minmax_idx_column_names = data.getMinMaxColumnsNames(metadata_snapshot->getPartitionKey()); + auto minmax_idx_column_names = MergeTreeData::getMinMaxColumnsNames(metadata_snapshot->getPartitionKey()); date_column = minmax_idx_column_names[data.minmax_idx_date_column_pos]; } diff --git a/src/Storages/StorageFuzzJSON.cpp b/src/Storages/StorageFuzzJSON.cpp index 87790dd2fdc..918f54b1672 100644 --- a/src/Storages/StorageFuzzJSON.cpp +++ b/src/Storages/StorageFuzzJSON.cpp @@ -364,7 +364,7 @@ JSONNode & fuzzSingleJSONNode(JSONNode & n, const StorageFuzzJSON::Configuration if (val.fixed) val.fixed = generateRandomFixedValue(config, rnd); - else if (val.array && val.array->size() < config.max_array_size && node_count + val.array->size() < config.value_number_limit) + else if (val.array && val.array->size() < config.max_array_size && node_count + val.array->size() < StorageFuzzJSON::Configuration::value_number_limit) { if (val.array->empty()) val.array->push_back(generateRandomJSONNode(config, rnd, /*with_key*/ false, depth)); @@ -377,7 +377,7 @@ JSONNode & fuzzSingleJSONNode(JSONNode & n, const StorageFuzzJSON::Configuration } ++node_count; } - else if (val.object && val.object->size() < config.max_object_size && node_count + val.object->size() < config.value_number_limit) + else if (val.object && val.object->size() < config.max_object_size && node_count + val.object->size() < StorageFuzzJSON::Configuration::value_number_limit) { val.object->push_back(generateRandomJSONNode(config, rnd, /*with_key*/ true, depth)); ++node_count; @@ -619,11 +619,11 @@ void StorageFuzzJSON::processNamedCollectionResult(Configuration & configuration { configuration.max_output_length = collection.get("max_output_length"); - if (configuration.max_output_length < 2 || configuration.max_output_length > configuration.output_length_limit) + if (configuration.max_output_length < 2 || configuration.max_output_length > StorageFuzzJSON::Configuration::output_length_limit) throw Exception( ErrorCodes::BAD_ARGUMENTS, "The value of the 'max_output_length' argument must be within the interval [2, {}.]", - configuration.output_length_limit); + StorageFuzzJSON::Configuration::output_length_limit); } if (collection.has("max_nesting_level")) @@ -638,11 +638,11 @@ void StorageFuzzJSON::processNamedCollectionResult(Configuration & configuration if (collection.has("max_string_value_length")) { auto max_string_value_length = collection.get("max_string_value_length"); - if (max_string_value_length > configuration.output_length_limit) + if (max_string_value_length > StorageFuzzJSON::Configuration::output_length_limit) throw Exception( ErrorCodes::BAD_ARGUMENTS, "The value of the 'max_string_value_length' argument must be at most {}.", - configuration.output_length_limit); + StorageFuzzJSON::Configuration::output_length_limit); configuration.max_string_value_length = std::min(max_string_value_length, configuration.max_output_length); } @@ -650,11 +650,11 @@ void StorageFuzzJSON::processNamedCollectionResult(Configuration & configuration if (collection.has("max_key_length")) { auto max_key_length = collection.get("max_key_length"); - if (max_key_length > configuration.output_length_limit) + if (max_key_length > StorageFuzzJSON::Configuration::output_length_limit) throw Exception( ErrorCodes::BAD_ARGUMENTS, "The value of the 'max_key_length' argument must be less or equal than {}.", - configuration.output_length_limit); + StorageFuzzJSON::Configuration::output_length_limit); configuration.max_key_length = std::min(max_key_length, configuration.max_output_length); configuration.min_key_length = std::min(configuration.min_key_length, configuration.max_key_length); } diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 3d1ce76dff1..6e7ea32ee59 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -426,7 +426,7 @@ TTLTableDescription TTLTableDescription::parse(const String & str, const Columns ParserTTLExpressionList parser; ASTPtr ast = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); - FunctionNameNormalizer().visit(ast.get()); + FunctionNameNormalizer::visit(ast.get()); return getTTLForTableFromAST(ast, columns, context, primary_key, context->getSettingsRef().allow_suspicious_ttl_expressions); } From 2578ceddadf0271eac6c48b059bd46b873a5ae5d Mon Sep 17 00:00:00 2001 From: Joseph Redfern Date: Wed, 3 Apr 2024 21:09:47 +0100 Subject: [PATCH 0970/1165] Correct typo in "use_invironment_credentials" configuration option `use_environment_credentials was incorrectly specified as `use_invironment_credentials` --- docs/en/operations/storing-data.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index 9ffbb64c1ed..2c642dd2f0b 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -36,7 +36,7 @@ E.g. configuration option s3 https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -47,7 +47,7 @@ is equal to configuration (from `24.1`): s3 local https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -56,7 +56,7 @@ Configuration s3_plain https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -67,7 +67,7 @@ is equal to s3 plain https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -79,7 +79,7 @@ Example of full storage configuration will look like: s3 https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 @@ -105,7 +105,7 @@ Starting with 24.1 clickhouse version, it can also look like: s3 local https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 @@ -324,7 +324,7 @@ Configuration: s3_plain https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -337,7 +337,7 @@ Configuration: azure plain https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` From d905b83369931400bbd87faeef138fc68ac17455 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 3 Apr 2024 20:10:18 +0000 Subject: [PATCH 0971/1165] Fix something bad --- src/Dictionaries/FlatDictionary.cpp | 3 ++- src/Functions/decodeHTMLComponent.cpp | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index e3b1e8a84e2..7509af31fac 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -412,7 +412,8 @@ void FlatDictionary::blockToAttributes(const Block & block) { const auto keys_column = block.safeGetByPosition(0).column; - DictionaryKeysExtractor keys_extractor({ keys_column }, DictionaryKeysArenaHolder::getComplexKeyArena()); + DictionaryKeysArenaHolder arena_holder; + DictionaryKeysExtractor keys_extractor({ keys_column }, arena_holder.getComplexKeyArena()); /// NOLINT(readability-static-accessed-through-instance) size_t keys_size = keys_extractor.getKeysSize(); static constexpr size_t key_offset = 1; diff --git a/src/Functions/decodeHTMLComponent.cpp b/src/Functions/decodeHTMLComponent.cpp index cb6ba0b07e2..4db3c43f946 100644 --- a/src/Functions/decodeHTMLComponent.cpp +++ b/src/Functions/decodeHTMLComponent.cpp @@ -70,8 +70,7 @@ namespace const char * src_pos = src; const char * src_end = src + src_size; char * dst_pos = dst; - // perfect hashmap to lookup html character references - HTMLCharacterHash hash; + // to hold char seq for lookup, reuse it std::vector seq; while (true) From dda1a0b9f0d09777fb7d0cc79ca26d8a58f16476 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 3 Apr 2024 20:44:02 +0000 Subject: [PATCH 0972/1165] Also enable modernize-use-override --- .clang-tidy | 1 - 1 file changed, 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 13c1b116ead..c98bee71d1a 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -96,7 +96,6 @@ Checks: [ '-modernize-use-default-member-init', '-modernize-use-emplace', '-modernize-use-nodiscard', - '-modernize-use-override', '-modernize-use-trailing-return-type', '-performance-inefficient-string-concatenation', From 2c2f2b95263d69014154f442cabb4973ecc35ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 3 Apr 2024 22:59:59 +0200 Subject: [PATCH 0973/1165] Missing include --- src/IO/HTTPCommon.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 09f7724d613..6e1c886b9b0 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include From 3c61a7f4bfc30fa23c8c9a0e90cddb12b5617cda Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 3 Apr 2024 23:34:31 +0200 Subject: [PATCH 0974/1165] Update storing-data.md From c6aed8b7938ee78030d5a23740a3bc191e85ca7f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 3 Apr 2024 19:19:54 -0300 Subject: [PATCH 0975/1165] add optional port to regex --- src/IO/S3/URI.cpp | 2 +- src/IO/S3/URI.h | 2 +- src/IO/tests/gtest_s3_uri.cpp | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 02c77518ab6..0d498c1d743 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -40,7 +40,7 @@ URI::URI(const std::string & uri_) /// Case when AWS Private Link Interface is being used /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html - static const RE2 aws_private_link_style_pattern(R"(bucket\.vpce\-([a-z0-9\-.]+)\.vpce.amazonaws.com)"); + static const RE2 aws_private_link_style_pattern(R"(bucket\.vpce\-([a-z0-9\-.]+)\.vpce.amazonaws.com(:\d{1,5})?)"); /// Case when bucket name and key represented in path of S3 URL. /// E.g. (https://s3.region.amazonaws.com/bucket-name/key) diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h index 79f3da3fbbb..06b7d03aa8c 100644 --- a/src/IO/S3/URI.h +++ b/src/IO/S3/URI.h @@ -17,7 +17,7 @@ namespace DB::S3 * The following patterns are allowed: * s3://bucket/key * http(s)://endpoint/bucket/key - * TODO specify aws private link + * http(s)://bucket..s3..vpce.amazonaws.com<:port_number>/bucket_name/key */ struct URI { diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index 0a164b0dd61..9c1f7bd5219 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -94,6 +94,20 @@ const TestCase TestCases[] = { "b/c/d/e/f/g/h/i/j/data.json", "", false}, + // Zonal + {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com/root/nested/file.txt"), + "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com", + "root", + "nested/file.txt", + "", + false}, + // Non standard port + {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com:65535/root/nested/file.txt"), + "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com:65535", + "root", + "nested/file.txt", + "", + false}, }; class S3UriTest : public testing::TestWithParam From 414f3aebef07c4a8976931cf7c2db72c91e5bed9 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 7 Mar 2024 11:59:41 +0300 Subject: [PATCH 0976/1165] Context getGlobalTemporaryVolume use shared lock --- src/Interpreters/Context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 65fcd51529b..df80a60d6d3 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -971,7 +971,7 @@ Strings Context::getWarnings() const /// TODO: remove, use `getTempDataOnDisk` VolumePtr Context::getGlobalTemporaryVolume() const { - std::lock_guard lock(shared->mutex); + SharedLockGuard lock(shared->mutex); /// Calling this method we just bypass the `temp_data_on_disk` and write to the file on the volume directly. /// Volume is the same for `root_temp_data_on_disk` (always set) and `temp_data_on_disk` (if it's set). if (shared->root_temp_data_on_disk) From b91d446630a6054abcbc01251fba3abf032a62b6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 4 Apr 2024 10:57:29 +0200 Subject: [PATCH 0977/1165] Ping CI From d5229da1ec26918f8904eff0d91eabc8db89f1db Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 4 Apr 2024 09:09:03 +0000 Subject: [PATCH 0978/1165] Fix tidy build --- src/Functions/GatherUtils/Sources.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/GatherUtils/Sources.h b/src/Functions/GatherUtils/Sources.h index 01b63338559..4e3009a695d 100644 --- a/src/Functions/GatherUtils/Sources.h +++ b/src/Functions/GatherUtils/Sources.h @@ -144,7 +144,7 @@ struct NumericArraySource : public ArraySourceImpl> #pragma clang diagnostic ignored "-Wsuggest-override" #pragma clang diagnostic ignored "-Wsuggest-destructor-override" -/// NOLINTBEGIN(hicpp-use-override) +/// NOLINTBEGIN(hicpp-use-override, modernize-use-override) template struct ConstSource : public Base @@ -233,7 +233,7 @@ struct ConstSource : public Base } }; -/// NOLINTEND(hicpp-use-override) +/// NOLINTEND(hicpp-use-override, modernize-use-override) #pragma clang diagnostic pop From 5ce8ab2bd986d0c34abd7d6cf3cf028cad369cbb Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 4 Apr 2024 11:32:32 +0200 Subject: [PATCH 0979/1165] More logging --- src/Interpreters/Cache/FileCache.cpp | 10 ++++ src/Interpreters/Cache/FileCacheFactory.cpp | 5 +- src/Interpreters/Cache/FileCacheSettings.cpp | 58 ++++++++++++++++++++ src/Interpreters/Cache/FileCacheSettings.h | 3 + 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 90671629e64..8ab46e66a76 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1379,6 +1379,16 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, } } + { + auto cache_lock = lockCache(); + LOG_TRACE(log, "new max size: {}, old max size: {}, " + "new elements count: {}, old_elements_count: {}, " + "current max size: {}, current max elements: {}", + new_settings.max_size, actual_settings.max_size, + new_settings.max_elements, actual_settings.max_elements, + main_priority->getSizeLimit(cache_lock), main_priority->getElementsLimit(cache_lock)); + } + if (new_settings.max_size != actual_settings.max_size || new_settings.max_elements != actual_settings.max_elements) { diff --git a/src/Interpreters/Cache/FileCacheFactory.cpp b/src/Interpreters/Cache/FileCacheFactory.cpp index e1f14406058..a046c4c2b6a 100644 --- a/src/Interpreters/Cache/FileCacheFactory.cpp +++ b/src/Interpreters/Cache/FileCacheFactory.cpp @@ -162,7 +162,10 @@ void FileCacheFactory::updateSettingsFromConfig(const Poco::Util::AbstractConfig continue; } - LOG_TRACE(log, "Will apply settings changes for cache: {}", cache_name); + LOG_TRACE(log, "Will apply settings changes for cache {}. " + "Settings changes: {} (new settings: {}, old_settings: {})", + cache_name, fmt::join(new_settings.getSettingsDiff(old_settings), ", "), + new_settings.toString(), old_settings.toString()); try { diff --git a/src/Interpreters/Cache/FileCacheSettings.cpp b/src/Interpreters/Cache/FileCacheSettings.cpp index 8a48a2de68f..fef17d0df28 100644 --- a/src/Interpreters/Cache/FileCacheSettings.cpp +++ b/src/Interpreters/Cache/FileCacheSettings.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace DB @@ -98,4 +99,61 @@ void FileCacheSettings::loadFromCollection(const NamedCollection & collection) loadImpl(std::move(collection_has), std::move(collection_get_uint), std::move(collection_get_string), std::move(collection_get_double)); } +std::string FileCacheSettings::toString() const +{ + WriteBufferFromOwnString res; + res << "base_path: " << base_path << "\n"; + res << "max_size: " << max_size << "\n"; + res << "max_elements: " << max_elements << "\n"; + res << "max_file_segment_size: " << max_file_segment_size << "\n"; + res << "cache_on_write_operations: " << cache_on_write_operations << "\n"; + res << "cache_hits_threshold: " << cache_hits_threshold << "\n"; + res << "enable_filesystem_query_cache_limit: " << enable_filesystem_query_cache_limit << "\n"; + res << "bypass_cache_threshold: " << bypass_cache_threshold << "\n"; + res << "boundary_alignment: " << boundary_alignment << "\n"; + res << "background_download_threads: " << background_download_threads << "\n"; + res << "background_download_queue_size_limit: " << background_download_queue_size_limit << "\n"; + res << "load_metadata_threads: " << load_metadata_threads << "\n"; + res << "write_cache_per_user_id_directory: " << write_cache_per_user_id_directory << "\n"; + res << "cache_policy: " << cache_policy << "\n"; + res << "slru_size_ratio: " << slru_size_ratio << "\n"; + return res.str(); +} + +std::vector FileCacheSettings::getSettingsDiff(const FileCacheSettings & other) const +{ + std::vector res; + if (base_path != other.base_path) + res.push_back("base_path"); + if (max_size != other.max_size) + res.push_back("max_size"); + if (max_elements != other.max_elements) + res.push_back("max_elements"); + if (max_file_segment_size != other.max_file_segment_size) + res.push_back("max_file_segment_size"); + if (cache_on_write_operations != other.cache_on_write_operations) + res.push_back("cache_on_write_operations"); + if (cache_hits_threshold != other.cache_hits_threshold) + res.push_back("cache_hits_threshold"); + if (enable_filesystem_query_cache_limit != other.enable_filesystem_query_cache_limit) + res.push_back("enable_filesystem_query_cache_limit"); + if (bypass_cache_threshold != other.bypass_cache_threshold) + res.push_back("bypass_cache_threshold"); + if (boundary_alignment != other.boundary_alignment) + res.push_back("boundary_alignment"); + if (background_download_threads != other.background_download_threads) + res.push_back("background_download_threads"); + if (background_download_queue_size_limit != other.background_download_queue_size_limit) + res.push_back("background_download_queue_size_limit"); + if (load_metadata_threads != other.load_metadata_threads) + res.push_back("load_metadata_threads"); + if (write_cache_per_user_id_directory != other.write_cache_per_user_id_directory) + res.push_back("write_cache_per_user_directory"); + if (cache_policy != other.cache_policy) + res.push_back("cache_policy"); + if (slru_size_ratio != other.slru_size_ratio) + res.push_back("slru_size_ratio"); + return res; +} + } diff --git a/src/Interpreters/Cache/FileCacheSettings.h b/src/Interpreters/Cache/FileCacheSettings.h index 14770b3f005..7dab14ac896 100644 --- a/src/Interpreters/Cache/FileCacheSettings.h +++ b/src/Interpreters/Cache/FileCacheSettings.h @@ -41,6 +41,9 @@ struct FileCacheSettings void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); void loadFromCollection(const NamedCollection & collection); + std::string toString() const; + std::vector getSettingsDiff(const FileCacheSettings & other) const; + bool operator ==(const FileCacheSettings &) const = default; private: From e516bef844b397a8f9c041b914e85939b225446c Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 4 Apr 2024 09:52:58 +0000 Subject: [PATCH 0980/1165] Annalyzer: limit maximal size of column in constant folding --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f5474ddb662..8f6c461d92d 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -6083,7 +6083,9 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi * Example: SELECT toTypeName(sum(number)) FROM numbers(10); */ if (column && isColumnConst(*column) && !typeid_cast(column.get())->getDataColumn().isDummy() && - (!hasAggregateFunctionNodes(node) && !hasFunctionNode(node, "arrayJoin"))) + !hasAggregateFunctionNodes(node) && !hasFunctionNode(node, "arrayJoin") && + /// Sanity check: do not convert large columns to constants + column->byteSize() < 1_MiB) { /// Replace function node with result constant node Field column_constant_value; From 58c79af80b38c1c3b3f64718690edafdad2156af Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 4 Apr 2024 12:22:11 +0200 Subject: [PATCH 0981/1165] Simpler --- src/Common/QueryProfiler.cpp | 9 ++++++++ src/Common/ThreadStatus.cpp | 23 +++++++++++++++++++ .../Standalone/ThreadStatusExt.cpp | 4 ---- src/Interpreters/ThreadStatusExt.cpp | 21 ----------------- .../test_trace_collector_serverwide/test.py | 3 +++ 5 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index f985ec95e88..b616b795405 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -264,7 +264,16 @@ QueryProfilerBase::QueryProfilerBase(UInt64 thread_id, int clock_t template void QueryProfilerBase::setPeriod(UInt32 period_) { +#if defined(SANITIZER) + UNUSED(period); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QueryProfiler disabled because they cannot work under sanitizers"); +#elif defined(__APPLE__) + UNUSED(period); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QueryProfiler cannot work on OSX"); +#else timer.set(period_); +#endif + } template diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index cf50d305e95..a3a7257b472 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -124,6 +124,29 @@ ThreadStatus::ThreadStatus(bool check_current_thread_on_destruction_) #endif } +void ThreadStatus::initGlobalProfiler(UInt64 global_profiler_real_time_period, UInt64 global_profiler_cpu_time_period) +{ +#if !defined(SANITIZER) && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD) && !defined(__APPLE__) + try + { + if (global_profiler_real_time_period > 0) + query_profiler_real = std::make_unique(thread_id, + /* period= */ static_cast(global_profiler_real_time_period)); + + if (global_profiler_cpu_time_period > 0) + query_profiler_cpu = std::make_unique(thread_id, + /* period= */ static_cast(global_profiler_cpu_time_period)); + } + catch (...) + { + tryLogCurrentException("ThreadStatus", "Cannot initialize GlobalProfiler"); + } +#else + UNUSED(global_profiler_real_time_period); + UNUSED(global_profiler_cpu_time_period); +#endif +} + ThreadGroupPtr ThreadStatus::getThreadGroup() const { chassert(current_thread == this); diff --git a/src/Coordination/Standalone/ThreadStatusExt.cpp b/src/Coordination/Standalone/ThreadStatusExt.cpp index 2b89e2f024d..97f7287be8c 100644 --- a/src/Coordination/Standalone/ThreadStatusExt.cpp +++ b/src/Coordination/Standalone/ThreadStatusExt.cpp @@ -11,8 +11,4 @@ void CurrentThread::attachToGroup(const ThreadGroupPtr &) { } -void ThreadStatus::initGlobalProfiler(UInt64, UInt64) -{ -} - } diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 4b9bd069bc6..2b8e8bef6d4 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -458,27 +458,6 @@ void ThreadStatus::resetPerformanceCountersLastUsage() taskstats->reset(); } - -void ThreadStatus::initGlobalProfiler(UInt64 global_profiler_real_time_period, UInt64 global_profiler_cpu_time_period) -{ - - try - { - if (global_profiler_real_time_period > 0) - query_profiler_real = std::make_unique(thread_id, - /* period= */ static_cast(global_profiler_real_time_period)); - - if (global_profiler_cpu_time_period > 0) - query_profiler_cpu = std::make_unique(thread_id, - /* period= */ static_cast(global_profiler_cpu_time_period)); - } - catch (...) - { - tryLogCurrentException("ThreadStatus", "Cannot initialize GlobalProfiler"); - } - -} - void ThreadStatus::initQueryProfiler() { if (internal_thread) diff --git a/tests/integration/test_trace_collector_serverwide/test.py b/tests/integration/test_trace_collector_serverwide/test.py index 88d235642b9..9bd107ac365 100644 --- a/tests/integration/test_trace_collector_serverwide/test.py +++ b/tests/integration/test_trace_collector_serverwide/test.py @@ -22,6 +22,9 @@ def start_cluster(): def test_global_thread_profiler(start_cluster): + if node1.is_built_with_sanitizer(): + return + node1.query( "CREATE TABLE t (key UInt32, value String) Engine = MergeTree() ORDER BY key" ) From 276246ee9740a7c737434c8bdb0c8b33e19ee321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 4 Apr 2024 12:29:54 +0200 Subject: [PATCH 0982/1165] Introduce IAggregateFunction_fwd to reduce header dependencies --- src/AggregateFunctions/IAggregateFunction.h | 10 +- .../IAggregateFunction_fwd.h | 14 +++ src/Analyzer/ArrayJoinNode.cpp | 12 +-- src/Analyzer/ColumnNode.cpp | 10 +- src/Analyzer/JoinNode.cpp | 12 +-- src/Analyzer/WindowNode.cpp | 8 +- src/Columns/ColumnAggregateFunction.h | 8 +- src/DataTypes/DataTypeAggregateFunction.cpp | 25 +++++ src/DataTypes/DataTypeAggregateFunction.h | 17 ++-- .../DataTypeCustomSimpleAggregateFunction.cpp | 1 + .../DataTypeCustomSimpleAggregateFunction.h | 7 +- .../SerializationAggregateFunction.cpp | 1 + .../SerializationAggregateFunction.h | 2 +- src/Formats/ProtobufSerializer.cpp | 93 ++++++++++--------- src/Functions/FunctionBinaryArithmetic.h | 33 +++---- src/Functions/FunctionsConversion.cpp | 1 + src/Functions/runningAccumulate.cpp | 7 +- src/Interpreters/AggregateDescription.cpp | 4 +- src/Interpreters/AggregateDescription.h | 5 +- src/Interpreters/AggregatedData.h | 2 +- src/Interpreters/AggregatedDataVariants.h | 7 +- src/Interpreters/AggregationUtils.cpp | 1 + src/Interpreters/HashJoin.cpp | 1 + src/Interpreters/JIT/compileFunction.cpp | 19 ++-- src/Interpreters/JIT/compileFunction.h | 8 +- .../RewriteFunctionToSubcolumnVisitor.cpp | 7 +- src/Interpreters/WindowDescription.cpp | 8 +- src/Interpreters/WindowDescription.h | 8 +- src/Parsers/ExpressionElementParsers.cpp | 5 +- .../Algorithms/AggregatingSortedAlgorithm.h | 3 +- src/Processors/Merges/Algorithms/Graphite.cpp | 24 ++++- src/Processors/Merges/Algorithms/Graphite.h | 22 +---- src/Processors/QueryPlan/WindowStep.cpp | 12 +-- src/Processors/Transforms/WindowTransform.h | 2 + src/Storages/System/StorageSystemGraphite.cpp | 5 +- .../System/StorageSystemMySQLBinlogs.cpp | 10 +- .../System/StorageSystemRemoteDataPaths.cpp | 11 ++- 37 files changed, 236 insertions(+), 189 deletions(-) create mode 100644 src/AggregateFunctions/IAggregateFunction_fwd.h diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 97e0e89aee9..ee227db6d9d 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -1,17 +1,18 @@ #pragma once +#include #include #include #include #include #include #include +#include #include #include #include #include #include -#include #include "config.h" @@ -46,13 +47,6 @@ class IWindowFunction; using DataTypePtr = std::shared_ptr; using DataTypes = std::vector; -using AggregateDataPtr = char *; -using AggregateDataPtrs = std::vector; -using ConstAggregateDataPtr = const char *; - -class IAggregateFunction; -using AggregateFunctionPtr = std::shared_ptr; - struct AggregateFunctionProperties; /** Aggregate functions interface. diff --git a/src/AggregateFunctions/IAggregateFunction_fwd.h b/src/AggregateFunctions/IAggregateFunction_fwd.h new file mode 100644 index 00000000000..7c78e32c652 --- /dev/null +++ b/src/AggregateFunctions/IAggregateFunction_fwd.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +namespace DB +{ +using AggregateDataPtr = char *; +using AggregateDataPtrs = std::vector; +using ConstAggregateDataPtr = const char *; + +class IAggregateFunction; +using AggregateFunctionPtr = std::shared_ptr; +} diff --git a/src/Analyzer/ArrayJoinNode.cpp b/src/Analyzer/ArrayJoinNode.cpp index ee6bd80150d..5ae097b1c12 100644 --- a/src/Analyzer/ArrayJoinNode.cpp +++ b/src/Analyzer/ArrayJoinNode.cpp @@ -1,14 +1,12 @@ #include - +#include +#include +#include #include #include -#include - -#include #include - -#include -#include +#include +#include namespace DB { diff --git a/src/Analyzer/ColumnNode.cpp b/src/Analyzer/ColumnNode.cpp index 3d9f5d1640e..fa8c59f50f3 100644 --- a/src/Analyzer/ColumnNode.cpp +++ b/src/Analyzer/ColumnNode.cpp @@ -1,14 +1,12 @@ #include - -#include - +#include +#include #include #include -#include - #include +#include +#include -#include namespace DB { diff --git a/src/Analyzer/JoinNode.cpp b/src/Analyzer/JoinNode.cpp index 9b61c8b19d0..53a003ec3c0 100644 --- a/src/Analyzer/JoinNode.cpp +++ b/src/Analyzer/JoinNode.cpp @@ -1,16 +1,14 @@ #include #include - +#include +#include #include #include -#include - -#include -#include #include +#include +#include #include - -#include +#include namespace DB { diff --git a/src/Analyzer/WindowNode.cpp b/src/Analyzer/WindowNode.cpp index 0fbe7c51bc7..af830815fd5 100644 --- a/src/Analyzer/WindowNode.cpp +++ b/src/Analyzer/WindowNode.cpp @@ -1,11 +1,9 @@ #include - -#include - -#include #include - +#include #include +#include +#include namespace DB { diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index c5d854e208a..a75b27e835c 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -17,13 +18,6 @@ using ArenaPtr = std::shared_ptr; using ConstArenaPtr = std::shared_ptr; using ConstArenas = std::vector; -using AggregateDataPtr = char *; -using AggregateDataPtrs = std::vector; -using ConstAggregateDataPtr = const char *; - -class IAggregateFunction; -using AggregateFunctionPtr = std::shared_ptr; - class Context; using ContextPtr = std::shared_ptr; diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 14a3c6a4248..ef7d86d2a81 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -32,6 +33,11 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +String DataTypeAggregateFunction::getFunctionName() const +{ + return function->getName(); +} + String DataTypeAggregateFunction::doGetName() const { @@ -52,6 +58,25 @@ size_t DataTypeAggregateFunction::getVersion() const return function->getDefaultVersion(); } +DataTypePtr DataTypeAggregateFunction::getReturnType() const +{ + return function->getResultType(); +} + +DataTypePtr DataTypeAggregateFunction::getReturnTypeToPredict() const +{ + return function->getReturnTypeToPredict(); +} + +bool DataTypeAggregateFunction::isVersioned() const +{ + return function->isVersioned(); +} + +void DataTypeAggregateFunction::updateVersionFromRevision(size_t revision, bool if_empty) const +{ + setVersion(function->getVersionFromRevision(revision), if_empty); +} String DataTypeAggregateFunction::getNameImpl(bool with_version) const { diff --git a/src/DataTypes/DataTypeAggregateFunction.h b/src/DataTypes/DataTypeAggregateFunction.h index 7d1bb355ccf..8b4b3d6ee4c 100644 --- a/src/DataTypes/DataTypeAggregateFunction.h +++ b/src/DataTypes/DataTypeAggregateFunction.h @@ -1,7 +1,7 @@ #pragma once -#include - +#include +#include #include @@ -39,7 +39,7 @@ public: { } - String getFunctionName() const { return function->getName(); } + String getFunctionName() const; AggregateFunctionPtr getFunction() const { return function; } String doGetName() const override; @@ -51,8 +51,8 @@ public: bool canBeInsideNullable() const override { return false; } - DataTypePtr getReturnType() const { return function->getResultType(); } - DataTypePtr getReturnTypeToPredict() const { return function->getReturnTypeToPredict(); } + DataTypePtr getReturnType() const; + DataTypePtr getReturnTypeToPredict() const; DataTypes getArgumentsDataTypes() const { return argument_types; } MutableColumnPtr createColumn() const override; @@ -69,7 +69,7 @@ public: SerializationPtr doGetDefaultSerialization() const override; bool supportsSparseSerialization() const override { return false; } - bool isVersioned() const { return function->isVersioned(); } + bool isVersioned() const; /// Version is not empty only if it was parsed from AST or implicitly cast to 0 or version according /// to server revision. @@ -84,10 +84,7 @@ public: version = version_; } - void updateVersionFromRevision(size_t revision, bool if_empty) const - { - setVersion(function->getVersionFromRevision(revision), if_empty); - } + void updateVersionFromRevision(size_t revision, bool if_empty) const; }; void setVersionToAggregateFunctions(DataTypePtr & type, bool if_empty, std::optional revision = std::nullopt); diff --git a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp index ee9870eb0ef..cae9622bcb9 100644 --- a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp +++ b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.h b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.h index 926dfd9cc82..bdabb465fe5 100644 --- a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.h +++ b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.h @@ -1,13 +1,18 @@ #pragma once +#include +#include #include -#include #include namespace DB { +class IDataType; +using DataTypePtr = std::shared_ptr; +using DataTypes = std::vector; + /** The type SimpleAggregateFunction(fct, type) is meant to be used in an AggregatingMergeTree. It behaves like a standard * data type but when rows are merged, an aggregation function is applied. * diff --git a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp index 2ac23d52e28..640d2c419d4 100644 --- a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp +++ b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/DataTypes/Serializations/SerializationAggregateFunction.h b/src/DataTypes/Serializations/SerializationAggregateFunction.h index 4212298bbc1..c45fc79f714 100644 --- a/src/DataTypes/Serializations/SerializationAggregateFunction.h +++ b/src/DataTypes/Serializations/SerializationAggregateFunction.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index c0d0713e254..f2f1d985cc9 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -1,51 +1,54 @@ #include #if USE_PROTOBUF -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include +# include +# include +# include +# include +# include +# include +# include + namespace DB { diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 79e5ee442c2..89ff63995b1 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -5,54 +5,55 @@ // sanitizer/asan_interface.h #include #include -#include +#include #include +#include #include #include #include #include #include #include +#include +#include +#include #include #include +#include #include #include #include #include #include -#include -#include -#include #include +#include +#include +#include +#include #include #include +#include #include #include +#include #include #include #include #include #include #include +#include #include #include +#include #include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #if USE_EMBEDDED_COMPILER # include diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 0f624a2fa2e..9f994055afc 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Functions/runningAccumulate.cpp b/src/Functions/runningAccumulate.cpp index b0ba10c4049..793e79cdf46 100644 --- a/src/Functions/runningAccumulate.cpp +++ b/src/Functions/runningAccumulate.cpp @@ -1,8 +1,9 @@ -#include -#include -#include +#include #include #include +#include +#include +#include #include #include #include diff --git a/src/Interpreters/AggregateDescription.cpp b/src/Interpreters/AggregateDescription.cpp index 787e0a503f8..d4c09995b56 100644 --- a/src/Interpreters/AggregateDescription.cpp +++ b/src/Interpreters/AggregateDescription.cpp @@ -1,7 +1,7 @@ +#include +#include #include #include -#include - #include diff --git a/src/Interpreters/AggregateDescription.h b/src/Interpreters/AggregateDescription.h index 8c3302a8b0b..0f1c0ce67ae 100644 --- a/src/Interpreters/AggregateDescription.h +++ b/src/Interpreters/AggregateDescription.h @@ -1,13 +1,16 @@ #pragma once -#include +#include #include +#include #include #include namespace DB { +class WriteBuffer; + namespace JSONBuilder { class JSONMap; } struct AggregateDescription diff --git a/src/Interpreters/AggregatedData.h b/src/Interpreters/AggregatedData.h index 6cd6b190801..4b581c682ca 100644 --- a/src/Interpreters/AggregatedData.h +++ b/src/Interpreters/AggregatedData.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include diff --git a/src/Interpreters/AggregatedDataVariants.h b/src/Interpreters/AggregatedDataVariants.h index 8b82c5d9842..9f7185db9fc 100644 --- a/src/Interpreters/AggregatedDataVariants.h +++ b/src/Interpreters/AggregatedDataVariants.h @@ -1,11 +1,12 @@ #pragma once -#include -#include -#include #include #include #include +#include +#include + + namespace DB { class Arena; diff --git a/src/Interpreters/AggregationUtils.cpp b/src/Interpreters/AggregationUtils.cpp index 125a9e4f6b8..132ce93786a 100644 --- a/src/Interpreters/AggregationUtils.cpp +++ b/src/Interpreters/AggregationUtils.cpp @@ -1,3 +1,4 @@ +#include #include namespace DB diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 12a906526f6..73498b39ead 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index f50a122f9a2..31d3920ccfd 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -2,16 +2,17 @@ #if USE_EMBEDDED_COMPILER -#include -#include -#include +# include +# include +# include +# include +# include +# include +# include -#include -#include -#include -#include -#include -#include +# include +# include +# include namespace { diff --git a/src/Interpreters/JIT/compileFunction.h b/src/Interpreters/JIT/compileFunction.h index 84abfa0925a..551e4d0bb11 100644 --- a/src/Interpreters/JIT/compileFunction.h +++ b/src/Interpreters/JIT/compileFunction.h @@ -4,10 +4,10 @@ #if USE_EMBEDDED_COMPILER -#include -#include -#include -#include +# include +# include +# include +# include namespace DB diff --git a/src/Interpreters/RewriteFunctionToSubcolumnVisitor.cpp b/src/Interpreters/RewriteFunctionToSubcolumnVisitor.cpp index 0717abd4782..f0202199752 100644 --- a/src/Interpreters/RewriteFunctionToSubcolumnVisitor.cpp +++ b/src/Interpreters/RewriteFunctionToSubcolumnVisitor.cpp @@ -1,9 +1,10 @@ -#include -#include #include +#include +#include +#include #include #include -#include +#include namespace DB { diff --git a/src/Interpreters/WindowDescription.cpp b/src/Interpreters/WindowDescription.cpp index 8a7a5024d69..31a881001e3 100644 --- a/src/Interpreters/WindowDescription.cpp +++ b/src/Interpreters/WindowDescription.cpp @@ -1,10 +1,10 @@ -#include - +#include #include -#include -#include #include +#include #include +#include +#include namespace DB diff --git a/src/Interpreters/WindowDescription.h b/src/Interpreters/WindowDescription.h index d14908fe993..05269c9d2c3 100644 --- a/src/Interpreters/WindowDescription.h +++ b/src/Interpreters/WindowDescription.h @@ -1,12 +1,12 @@ #pragma once +#include #include -#include -#include -#include -#include #include +#include #include +#include +#include namespace DB { diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 67f4a306292..d5f8baf547e 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -5,9 +5,10 @@ #include #include -#include -#include #include +#include +#include +#include #include "Parsers/CommonParsers.h" #include diff --git a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h index aa221573151..db8ee66ab2b 100644 --- a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h @@ -1,9 +1,10 @@ #pragma once +#include #include -#include #include #include +#include namespace DB { diff --git a/src/Processors/Merges/Algorithms/Graphite.cpp b/src/Processors/Merges/Algorithms/Graphite.cpp index 817961b709c..e3e94b8dd27 100644 --- a/src/Processors/Merges/Algorithms/Graphite.cpp +++ b/src/Processors/Merges/Algorithms/Graphite.cpp @@ -1,16 +1,17 @@ -#include -#include #include +#include #include #include +#include #include +#include +#include #include #include #include #include -#include #include @@ -61,6 +62,23 @@ RuleType ruleType(const String & s) throw Exception(DB::ErrorCodes::BAD_ARGUMENTS, "invalid rule type: {}", s); } +void Pattern::updateHash(SipHash & hash) const +{ + hash.update(rule_type); + hash.update(regexp_str); + if (function) + { + hash.update(function->getName()); + for (const auto & p : function->getParameters()) + hash.update(toString(p)); + } + for (const auto & r : retentions) + { + hash.update(r.age); + hash.update(r.precision); + } +} + static const Graphite::Pattern undef_pattern = { /// empty pattern for selectPatternForPath .rule_type = RuleTypeAll, diff --git a/src/Processors/Merges/Algorithms/Graphite.h b/src/Processors/Merges/Algorithms/Graphite.h index 04bb4548c14..ce3331053d1 100644 --- a/src/Processors/Merges/Algorithms/Graphite.h +++ b/src/Processors/Merges/Algorithms/Graphite.h @@ -1,8 +1,9 @@ #pragma once -#include +#include +#include #include -#include +#include /** Intended for implementation of "rollup" - aggregation (rounding) of older data * for a table with Graphite data (Graphite is the system for time series monitoring). @@ -123,22 +124,7 @@ struct Pattern AggregateFunctionPtr function; Retentions retentions; /// Must be ordered by 'age' descending. enum { TypeUndef, TypeRetention, TypeAggregation, TypeAll } type = TypeAll; /// The type of defined pattern, filled automatically - void updateHash(SipHash & hash) const - { - hash.update(rule_type); - hash.update(regexp_str); - if (function) - { - hash.update(function->getName()); - for (const auto & p : function->getParameters()) - hash.update(toString(p)); - } - for (const auto & r : retentions) - { - hash.update(r.age); - hash.update(r.precision); - } - } + void updateHash(SipHash & hash) const; }; bool operator==(const Pattern & a, const Pattern & b); diff --git a/src/Processors/QueryPlan/WindowStep.cpp b/src/Processors/QueryPlan/WindowStep.cpp index bb4f429d626..3d1faf7c079 100644 --- a/src/Processors/QueryPlan/WindowStep.cpp +++ b/src/Processors/QueryPlan/WindowStep.cpp @@ -1,10 +1,10 @@ -#include - -#include -#include -#include -#include +#include #include +#include +#include +#include +#include +#include #include namespace DB diff --git a/src/Processors/Transforms/WindowTransform.h b/src/Processors/Transforms/WindowTransform.h index 347c2516230..43fa6b28019 100644 --- a/src/Processors/Transforms/WindowTransform.h +++ b/src/Processors/Transforms/WindowTransform.h @@ -21,6 +21,8 @@ using ExpressionActionsPtr = std::shared_ptr; class Arena; +class IWindowFunction; + // Runtime data for computing one window function. struct WindowFunctionWorkspace { diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index d8b760e1302..ef13c3c24da 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -1,7 +1,8 @@ -#include -#include +#include #include #include +#include +#include namespace DB diff --git a/src/Storages/System/StorageSystemMySQLBinlogs.cpp b/src/Storages/System/StorageSystemMySQLBinlogs.cpp index 32648d22ee8..846fe3547d0 100644 --- a/src/Storages/System/StorageSystemMySQLBinlogs.cpp +++ b/src/Storages/System/StorageSystemMySQLBinlogs.cpp @@ -1,11 +1,11 @@ -#include - -#include -#include #include -#include +#include #include +#include +#include #include +#include +#include namespace DB diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index 0ca76430ceb..f54fa220e83 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -1,14 +1,15 @@ #include "StorageSystemRemoteDataPaths.h" -#include +#include +#include +#include #include +#include #include +#include #include #include -#include -#include -#include #include -#include +#include namespace fs = std::filesystem; From 33aee0f599867da294cfc5327cc4ab932e761066 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 4 Apr 2024 13:00:51 +0200 Subject: [PATCH 0983/1165] Analyzer: Fix name resolution from parent scopes --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 8 +++++- ...alyzer_resolve_from_parent_scope.reference | 1 + ...033_analyzer_resolve_from_parent_scope.sql | 27 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03033_analyzer_resolve_from_parent_scope.reference create mode 100644 tests/queries/0_stateless/03033_analyzer_resolve_from_parent_scope.sql diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f5474ddb662..91832f6060d 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -3993,9 +3993,15 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierInParentScopes(const } else if (resolved_identifier->as()) { - lookup_result.resolved_identifier = resolved_identifier; return lookup_result; } + else if (auto * resolved_function = resolved_identifier->as()) + { + /// Special case: scalar subquery was executed and replaced by __getScalar function. + /// Handle it as a constant. + if (resolved_function->getFunctionName() == "__getScalar") + return lookup_result; + } throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Resolve identifier '{}' from parent scope only supported for constants and CTE. Actual {} node type {}. In scope {}", diff --git a/tests/queries/0_stateless/03033_analyzer_resolve_from_parent_scope.reference b/tests/queries/0_stateless/03033_analyzer_resolve_from_parent_scope.reference new file mode 100644 index 00000000000..f599e28b8ab --- /dev/null +++ b/tests/queries/0_stateless/03033_analyzer_resolve_from_parent_scope.reference @@ -0,0 +1 @@ +10 diff --git a/tests/queries/0_stateless/03033_analyzer_resolve_from_parent_scope.sql b/tests/queries/0_stateless/03033_analyzer_resolve_from_parent_scope.sql new file mode 100644 index 00000000000..22f103c9bd5 --- /dev/null +++ b/tests/queries/0_stateless/03033_analyzer_resolve_from_parent_scope.sql @@ -0,0 +1,27 @@ +CREATE TABLE vecs_Float32 (v Array(Float32)) ENGINE=Memory; +INSERT INTO vecs_Float32 +SELECT v FROM ( + SELECT + number AS n, + [ + rand(n*10), rand(n*10+1), rand(n*10+2), rand(n*10+3), rand(n*10+4), rand(n*10+5), rand(n*10+6), rand(n*10+7), rand(n*10+8), rand(n*10+9), + rand(n*10+10), rand(n*10+11), rand(n*10+12), rand(n*10+13), rand(n*10+14), rand(n*10+15), rand(n*10+16), rand(n*10+17), rand(n*10+18), rand(n*10+19), + rand(n*10+20), rand(n*10+21), rand(n*10+22), rand(n*10+23), rand(n*10+24), rand(n*10+25), rand(n*10+26), rand(n*10+27), rand(n*10+28), rand(n*10+29), + rand(n*10+30), rand(n*10+31), rand(n*10+32), rand(n*10+33), rand(n*10+34), rand(n*10+35), rand(n*10+36), rand(n*10+37), rand(n*10+38), rand(n*10+39), + rand(n*10+40), rand(n*10+41), rand(n*10+42), rand(n*10+43), rand(n*10+44), rand(n*10+45), rand(n*10+46), rand(n*10+47), rand(n*10+48), rand(n*10+49), + rand(n*10+50), rand(n*10+51), rand(n*10+52), rand(n*10+53), rand(n*10+54), rand(n*10+55), rand(n*10+56), rand(n*10+57), rand(n*10+58), rand(n*10+59), + rand(n*10+60), rand(n*10+61), rand(n*10+62), rand(n*10+63), rand(n*10+64), rand(n*10+65), rand(n*10+66), rand(n*10+67), rand(n*10+68), rand(n*10+69), + rand(n*10+70), rand(n*10+71), rand(n*10+72), rand(n*10+73), rand(n*10+74), rand(n*10+75), rand(n*10+76), rand(n*10+77), rand(n*10+78), rand(n*10+79), + rand(n*10+80), rand(n*10+81), rand(n*10+82), rand(n*10+83), rand(n*10+84), rand(n*10+85), rand(n*10+86), rand(n*10+87), rand(n*10+88), rand(n*10+89), + rand(n*10+90), rand(n*10+91), rand(n*10+92), rand(n*10+93), rand(n*10+94), rand(n*10+95), rand(n*10+96), rand(n*10+97), rand(n*10+98), rand(n*10+99), + rand(n*10+100), rand(n*10+101), rand(n*10+102), rand(n*10+103), rand(n*10+104), rand(n*10+105), rand(n*10+106), rand(n*10+107), rand(n*10+108), rand(n*10+109), + rand(n*10+110), rand(n*10+111), rand(n*10+112), rand(n*10+113), rand(n*10+114), rand(n*10+115), rand(n*10+116), rand(n*10+117), rand(n*10+118), rand(n*10+119), + rand(n*10+120), rand(n*10+121), rand(n*10+122), rand(n*10+123), rand(n*10+124), rand(n*10+125), rand(n*10+126), rand(n*10+127), rand(n*10+128), rand(n*10+129), + rand(n*10+130), rand(n*10+131), rand(n*10+132), rand(n*10+133), rand(n*10+134), rand(n*10+135), rand(n*10+136), rand(n*10+137), rand(n*10+138), rand(n*10+139), + rand(n*10+140), rand(n*10+141), rand(n*10+142), rand(n*10+143), rand(n*10+144), rand(n*10+145), rand(n*10+146), rand(n*10+147), rand(n*10+148), rand(n*10+149) + ] AS v + FROM system.numbers + LIMIT 10 +); + +WITH (SELECT v FROM vecs_Float32 limit 1) AS a SELECT count(dp) FROM (SELECT dotProduct(a, v) AS dp FROM vecs_Float32); From 5d677936098c380c6f86f280c214b576f998175b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 4 Apr 2024 11:41:28 +0000 Subject: [PATCH 0984/1165] Fix docs about default value of `output_format_pretty_row_numbers` --- docs/en/interfaces/formats.md | 4 ++-- docs/en/operations/settings/settings-formats.md | 2 +- docs/ru/operations/settings/settings.md | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index b6235fd1182..ddbbb365e8a 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1487,7 +1487,7 @@ Differs from [PrettySpaceNoEscapes](#prettyspacenoescapes) in that up to 10,000 - [output_format_pretty_max_value_width](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_max_value_width) - Maximum width of value to display in Pretty formats. If greater - it will be cut. Default value - `10000`. - [output_format_pretty_color](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_color) - use ANSI escape sequences to paint colors in Pretty formats. Default value - `true`. - [output_format_pretty_grid_charset](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_grid_charset) - Charset for printing grid borders. Available charsets: ASCII, UTF-8. Default value - `UTF-8`. -- [output_format_pretty_row_numbers](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_row_numbers) - Add row numbers before each row for pretty output format. Default value - `false`. +- [output_format_pretty_row_numbers](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_row_numbers) - Add row numbers before each row for pretty output format. Default value - `true`. ## RowBinary {#rowbinary} @@ -2465,7 +2465,7 @@ Result: ## Npy {#data-format-npy} -This function is designed to load a NumPy array from a .npy file into ClickHouse. The NumPy file format is a binary format used for efficiently storing arrays of numerical data. During import, ClickHouse treats top level dimension as an array of rows with single column. Supported Npy data types and their corresponding type in ClickHouse: +This function is designed to load a NumPy array from a .npy file into ClickHouse. The NumPy file format is a binary format used for efficiently storing arrays of numerical data. During import, ClickHouse treats top level dimension as an array of rows with single column. Supported Npy data types and their corresponding type in ClickHouse: | Npy type | ClickHouse type | |:--------:|:---------------:| | b1 | UInt8 | diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 831c7094114..f455fcba840 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -1642,7 +1642,7 @@ Possible values: - 0 — Output without row numbers. - 1 — Output with row numbers. -Default value: `0`. +Default value: `1`. **Example** diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index a56afda641b..f9456e34a56 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2776,7 +2776,7 @@ SELECT range(number) FROM system.numbers LIMIT 5 FORMAT PrettyCompactNoEscapes; - 0 — номера строк не выводятся. - 1 — номера строк выводятся. -Значение по умолчанию: `0`. +Значение по умолчанию: `1`. **Пример** @@ -2798,7 +2798,7 @@ SELECT TOP 3 name, value FROM system.settings; ``` ### output_format_pretty_color {#output_format_pretty_color} -Включает/выключает управляющие последовательности ANSI в форматах Pretty. +Включает/выключает управляющие последовательности ANSI в форматах Pretty. Возможные значения: @@ -4123,7 +4123,7 @@ SELECT sum(number) FROM numbers(10000000000) SETTINGS partial_result_on_first_ca ## session_timezone {#session_timezone} Задаёт значение часового пояса (session_timezone) по умолчанию для текущей сессии вместо [часового пояса сервера](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone). То есть, все значения DateTime/DateTime64, для которых явно не задан часовой пояс, будут интерпретированы как относящиеся к указанной зоне. -При значении настройки `''` (пустая строка), будет совпадать с часовым поясом сервера. +При значении настройки `''` (пустая строка), будет совпадать с часовым поясом сервера. Функции `timeZone()` and `serverTimezone()` возвращают часовой пояс текущей сессии и сервера соответственно. From 62f9be052c3a1046fb492a313ad77df80e532009 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 4 Apr 2024 13:41:30 +0200 Subject: [PATCH 0985/1165] Fix test --- src/Interpreters/Cache/FileCache.cpp | 10 ------- src/Interpreters/Cache/FileCacheFactory.cpp | 8 +++--- src/Interpreters/Cache/FileCacheSettings.cpp | 30 ++++++++++---------- tests/config/config.d/storage_conf.xml | 2 +- 4 files changed, 20 insertions(+), 30 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 8ab46e66a76..90671629e64 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1379,16 +1379,6 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, } } - { - auto cache_lock = lockCache(); - LOG_TRACE(log, "new max size: {}, old max size: {}, " - "new elements count: {}, old_elements_count: {}, " - "current max size: {}, current max elements: {}", - new_settings.max_size, actual_settings.max_size, - new_settings.max_elements, actual_settings.max_elements, - main_priority->getSizeLimit(cache_lock), main_priority->getElementsLimit(cache_lock)); - } - if (new_settings.max_size != actual_settings.max_size || new_settings.max_elements != actual_settings.max_elements) { diff --git a/src/Interpreters/Cache/FileCacheFactory.cpp b/src/Interpreters/Cache/FileCacheFactory.cpp index a046c4c2b6a..747b31bff64 100644 --- a/src/Interpreters/Cache/FileCacheFactory.cpp +++ b/src/Interpreters/Cache/FileCacheFactory.cpp @@ -162,10 +162,10 @@ void FileCacheFactory::updateSettingsFromConfig(const Poco::Util::AbstractConfig continue; } - LOG_TRACE(log, "Will apply settings changes for cache {}. " - "Settings changes: {} (new settings: {}, old_settings: {})", - cache_name, fmt::join(new_settings.getSettingsDiff(old_settings), ", "), - new_settings.toString(), old_settings.toString()); + // LOG_TRACE(log, "Will apply settings changes for cache {}. " + // "Settings changes: {} (new settings: {}, old_settings: {})", + // cache_name, fmt::join(new_settings.getSettingsDiff(old_settings), ", "), + // new_settings.toString(), old_settings.toString()); try { diff --git a/src/Interpreters/Cache/FileCacheSettings.cpp b/src/Interpreters/Cache/FileCacheSettings.cpp index fef17d0df28..ff5f48503b7 100644 --- a/src/Interpreters/Cache/FileCacheSettings.cpp +++ b/src/Interpreters/Cache/FileCacheSettings.cpp @@ -102,21 +102,21 @@ void FileCacheSettings::loadFromCollection(const NamedCollection & collection) std::string FileCacheSettings::toString() const { WriteBufferFromOwnString res; - res << "base_path: " << base_path << "\n"; - res << "max_size: " << max_size << "\n"; - res << "max_elements: " << max_elements << "\n"; - res << "max_file_segment_size: " << max_file_segment_size << "\n"; - res << "cache_on_write_operations: " << cache_on_write_operations << "\n"; - res << "cache_hits_threshold: " << cache_hits_threshold << "\n"; - res << "enable_filesystem_query_cache_limit: " << enable_filesystem_query_cache_limit << "\n"; - res << "bypass_cache_threshold: " << bypass_cache_threshold << "\n"; - res << "boundary_alignment: " << boundary_alignment << "\n"; - res << "background_download_threads: " << background_download_threads << "\n"; - res << "background_download_queue_size_limit: " << background_download_queue_size_limit << "\n"; - res << "load_metadata_threads: " << load_metadata_threads << "\n"; - res << "write_cache_per_user_id_directory: " << write_cache_per_user_id_directory << "\n"; - res << "cache_policy: " << cache_policy << "\n"; - res << "slru_size_ratio: " << slru_size_ratio << "\n"; + res << "base_path: " << base_path << ", "; + res << "max_size: " << max_size << ", "; + res << "max_elements: " << max_elements << ", "; + res << "max_file_segment_size: " << max_file_segment_size << ", "; + res << "cache_on_write_operations: " << cache_on_write_operations << ", "; + res << "cache_hits_threshold: " << cache_hits_threshold << ", "; + res << "enable_filesystem_query_cache_limit: " << enable_filesystem_query_cache_limit << ", "; + res << "bypass_cache_threshold: " << bypass_cache_threshold << ", "; + res << "boundary_alignment: " << boundary_alignment << ", "; + res << "background_download_threads: " << background_download_threads << ", "; + res << "background_download_queue_size_limit: " << background_download_queue_size_limit << ", "; + res << "load_metadata_threads: " << load_metadata_threads << ", "; + res << "write_cache_per_user_id_directory: " << write_cache_per_user_id_directory << ", "; + res << "cache_policy: " << cache_policy << ", "; + res << "slru_size_ratio: " << slru_size_ratio << ", "; return res.str(); } diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml index 00d8cb3aea5..d40854247cd 100644 --- a/tests/config/config.d/storage_conf.xml +++ b/tests/config/config.d/storage_conf.xml @@ -19,7 +19,7 @@ cache s3_disk s3_cache/ - 64Mi + 104857600 1 100 LRU From 7e608f567b4794c7b28007c1d454ec5db5ec9657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 4 Apr 2024 13:55:52 +0200 Subject: [PATCH 0986/1165] Fix argMax with nullable non native numeric column --- src/AggregateFunctions/SingleValueData.cpp | 2 +- ...rgMinMax_numeric_non_extreme_bug.reference | 12 +++++++++ ...3035_argMinMax_numeric_non_extreme_bug.sql | 26 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03035_argMinMax_numeric_non_extreme_bug.reference create mode 100644 tests/queries/0_stateless/03035_argMinMax_numeric_non_extreme_bug.sql diff --git a/src/AggregateFunctions/SingleValueData.cpp b/src/AggregateFunctions/SingleValueData.cpp index 72eaf36e254..a14caf00f73 100644 --- a/src/AggregateFunctions/SingleValueData.cpp +++ b/src/AggregateFunctions/SingleValueData.cpp @@ -579,7 +579,7 @@ std::optional SingleValueDataFixed::getGreatestIndexNotNullIf( return std::nullopt; for (size_t i = index + 1; i < row_end; i++) - if ((!if_map || if_map[i] != 0) && (!null_map || null_map[i] == 0) && (vec[i] < vec[index])) + if ((!if_map || if_map[i] != 0) && (!null_map || null_map[i] == 0) && (vec[i] > vec[index])) index = i; return {index}; } diff --git a/tests/queries/0_stateless/03035_argMinMax_numeric_non_extreme_bug.reference b/tests/queries/0_stateless/03035_argMinMax_numeric_non_extreme_bug.reference new file mode 100644 index 00000000000..be07c950fea --- /dev/null +++ b/tests/queries/0_stateless/03035_argMinMax_numeric_non_extreme_bug.reference @@ -0,0 +1,12 @@ +Row 1: +────── +max(time): 2021-01-01 00:00:59.000 +max(toNullable(time)): 2021-01-01 00:00:59.000 +min(time): 2021-01-01 00:00:00.000 +min(toNullable(time)): 2021-01-01 00:00:00.000 +argMax(value, time): -1 +argMax(value, toNullable(time)): -1 +argMin(value, time): 0 +argMin(value, toNullable(time)): 0 +argMinIf(value, toNullable(time), notEquals(time, '2021-01-01 00:00:00.000')): 1 +argMaxIf(value, toNullable(time), notEquals(time, '2021-01-01 00:00:59.000')): -2 diff --git a/tests/queries/0_stateless/03035_argMinMax_numeric_non_extreme_bug.sql b/tests/queries/0_stateless/03035_argMinMax_numeric_non_extreme_bug.sql new file mode 100644 index 00000000000..deb580b9040 --- /dev/null +++ b/tests/queries/0_stateless/03035_argMinMax_numeric_non_extreme_bug.sql @@ -0,0 +1,26 @@ +CREATE TABLE IF NOT EXISTS test +( + `value` Float64 CODEC(Delta, LZ4), + `uuid` LowCardinality(String), + `time` DateTime64(3, 'UTC') CODEC(DoubleDelta, LZ4) +) +ENGINE = MergeTree() +ORDER BY uuid; + + +INSERT INTO test (uuid, time, value) +VALUES ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:00.000',0), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:09.000',1), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:10.000',2), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:19.000',3), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:20.000',2), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:29.000',1), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:30.000',0), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:39.000',-1), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:40.000',-2), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:49.000',-3), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:50.000',-2), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:00:59.000',-1), ('a1000000-0000-0000-0000-0000000000a1','2021-01-01 00:01:00.000',0); + +SELECT + max(time), + max(toNullable(time)), + min(time), + min(toNullable(time)), + argMax(value, time), + argMax(value, toNullable(time)), + argMin(value, time), + argMin(value, toNullable(time)), + argMinIf(value, toNullable(time), time != '2021-01-01 00:00:00.000'), + argMaxIf(value, toNullable(time), time != '2021-01-01 00:00:59.000'), +FROM test +WHERE (time >= fromUnixTimestamp64Milli(1609459200000, 'UTC')) AND (time < fromUnixTimestamp64Milli(1609459260000, 'UTC')) FORMAT Vertical; From a0925e6bc4efcbb0d9c3e8e8475810a566c63fa1 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:19:19 +0200 Subject: [PATCH 0987/1165] empty commit From b7c0501ac292f64992a06a4b880520174cf3cf00 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 4 Apr 2024 09:26:37 -0300 Subject: [PATCH 0988/1165] fix ut --- src/IO/tests/gtest_s3_uri.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index 9c1f7bd5219..175550acccc 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -103,7 +103,7 @@ const TestCase TestCases[] = { false}, // Non standard port {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com:65535/root/nested/file.txt"), - "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com:65535", + "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com:65535", "root", "nested/file.txt", "", From 9e1d8b8aaa438d7280c10f966102fab2aa1ae46b Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 4 Apr 2024 15:22:09 +0200 Subject: [PATCH 0989/1165] Fix mortonEncode with no rows --- src/Functions/mortonEncode.cpp | 3 +++ .../queries/0_stateless/03035_morton_encode_no_rows.reference | 2 ++ tests/queries/0_stateless/03035_morton_encode_no_rows.sql | 2 ++ 3 files changed, 7 insertions(+) create mode 100644 tests/queries/0_stateless/03035_morton_encode_no_rows.reference create mode 100644 tests/queries/0_stateless/03035_morton_encode_no_rows.sql diff --git a/src/Functions/mortonEncode.cpp b/src/Functions/mortonEncode.cpp index fee14c7784b..3b95c114b14 100644 --- a/src/Functions/mortonEncode.cpp +++ b/src/Functions/mortonEncode.cpp @@ -321,6 +321,9 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { + if (input_rows_count == 0) + return ColumnUInt64::create(); + return selector.selectAndExecute(arguments, result_type, input_rows_count); } diff --git a/tests/queries/0_stateless/03035_morton_encode_no_rows.reference b/tests/queries/0_stateless/03035_morton_encode_no_rows.reference new file mode 100644 index 00000000000..dc8bb19a028 --- /dev/null +++ b/tests/queries/0_stateless/03035_morton_encode_no_rows.reference @@ -0,0 +1,2 @@ +4294967286 +4294967286 diff --git a/tests/queries/0_stateless/03035_morton_encode_no_rows.sql b/tests/queries/0_stateless/03035_morton_encode_no_rows.sql new file mode 100644 index 00000000000..2663b1ac212 --- /dev/null +++ b/tests/queries/0_stateless/03035_morton_encode_no_rows.sql @@ -0,0 +1,2 @@ +SELECT mortonEncode(materialize((1, 1)), 65534, 65533); +SELECT mortonEncode((1, 1), 65534, 65533); From 01fecadabf959bc71377291d5a7cb8eb9ae9048c Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:35:51 +0200 Subject: [PATCH 0990/1165] Mark format RowBinaryWithDefaults as not suitable for output --- docs/en/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index b6235fd1182..37127b3ace4 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -79,7 +79,7 @@ The supported formats are: | [RowBinary](#rowbinary) | ✔ | ✔ | | [RowBinaryWithNames](#rowbinarywithnamesandtypes) | ✔ | ✔ | | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [RowBinaryWithDefaults](#rowbinarywithdefaults) | ✔ | ✔ | +| [RowBinaryWithDefaults](#rowbinarywithdefaults) | ✔ | ✗ | | [Native](#native) | ✔ | ✔ | | [Null](#null) | ✗ | ✔ | | [XML](#xml) | ✗ | ✔ | From 368dcf03ecc9426497aaaf739e9efa5fb43e1f63 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Thu, 4 Apr 2024 15:37:38 +0200 Subject: [PATCH 0991/1165] Run new analyzer tests with enabled allow_experimental_analyzer --- tests/queries/0_stateless/03033_cte_numbers_memory.sql | 1 + tests/queries/0_stateless/03033_with_fill_interpolate.sql | 1 + tests/queries/0_stateless/03034_normalized_ast.sql | 1 + tests/queries/0_stateless/03035_alias_column_bug_distributed.sql | 1 + tests/queries/0_stateless/03036_with_numbers.sql | 1 + tests/queries/0_stateless/03037_union_view.sql | 1 + tests/queries/0_stateless/03038_ambiguous_column.sql | 1 + .../0_stateless/03039_unknown_identifier_window_function.sql | 1 + tests/queries/0_stateless/03040_alias_column_join.sql | 1 + tests/queries/0_stateless/03040_array_sum_and_join.sql | 1 + tests/queries/0_stateless/03041_analyzer_gigachad_join.sql | 1 + tests/queries/0_stateless/03041_select_with_query_result.sql | 1 + tests/queries/0_stateless/03042_analyzer_alias_join.sql | 1 + tests/queries/0_stateless/03042_not_found_column_c1.sql | 1 + .../queries/0_stateless/03043_group_array_result_is_expected.sql | 1 + tests/queries/0_stateless/03044_analyzer_alias_join.sql | 1 + .../0_stateless/03044_array_join_columns_in_nested_table.sql | 1 + tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql | 1 + .../0_stateless/03045_unknown_identifier_alias_substitution.sql | 1 + tests/queries/0_stateless/03046_column_in_block_array_join.sql | 1 + tests/queries/0_stateless/03047_analyzer_alias_join.sql | 1 + .../0_stateless/03047_group_by_field_identified_aggregation.sql | 1 + .../queries/0_stateless/03048_not_found_column_xxx_in_block.sql | 1 + tests/queries/0_stateless/03049_analyzer_group_by_alias.sql | 1 + .../0_stateless/03049_unknown_identifier_materialized_column.sql | 1 + tests/queries/0_stateless/03050_select_one_one_one.sql | 1 + tests/queries/0_stateless/03051_many_ctes.sql | 1 + tests/queries/0_stateless/03052_query_hash_includes_aliases.sql | 1 + tests/queries/0_stateless/03053_analyzer_join_alias.sql | 1 + tests/queries/0_stateless/03054_analyzer_join_alias.sql | 1 + .../queries/0_stateless/03055_analyzer_subquery_group_array.sql | 1 + .../queries/0_stateless/03056_analyzer_double_subquery_alias.sql | 1 + tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql | 1 + tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql | 1 + .../0_stateless/03059_analyzer_join_engine_missing_column.sql | 1 + tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql | 1 + .../0_stateless/03061_analyzer_alias_as_right_key_in_join.sql | 1 + .../0_stateless/03062_analyzer_join_engine_missing_column.sql | 1 + .../03063_analyzer_multi_join_wrong_table_specifier.sql | 1 + tests/queries/0_stateless/03064_analyzer_named_subqueries.sql | 1 + .../0_stateless/03065_analyzer_cross_join_and_array_join.sql | 1 + .../queries/0_stateless/03066_analyzer_global_with_statement.sql | 1 + tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql | 1 + tests/queries/0_stateless/03068_analyzer_distributed_join.sql | 1 + .../0_stateless/03069_analyzer_with_alias_in_array_join.sql | 1 + .../queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql | 1 + .../03071_analyzer_array_join_forbid_non_existing_columns.sql | 1 + .../0_stateless/03072_analyzer_missing_columns_from_subquery.sql | 1 + .../queries/0_stateless/03073_analyzer_alias_as_column_name.sql | 1 + .../queries/0_stateless/03074_analyzer_alias_column_in_view.sql | 1 + tests/queries/0_stateless/03075_analyzer_subquery_alias.sql | 1 + .../queries/0_stateless/03076_analyzer_multiple_joins_alias.sql | 1 + .../0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql | 1 + .../0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql | 1 + .../03079_analyzer_numeric_literals_as_column_names.sql | 1 + ...080_analyzer_prefer_column_name_to_alias__virtual_columns.sql | 1 + tests/queries/0_stateless/03080_incorrect_join_with_merge.sql | 1 + tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql | 1 + .../0_stateless/03082_analyzer_left_join_correct_column.sql | 1 + tests/queries/0_stateless/03084_analyzer_join_column_alias.sql | 1 + .../queries/0_stateless/03085_analyzer_alias_column_group_by.sql | 1 + .../0_stateless/03086_analyzer_window_func_part_of_group_by.sql | 1 + tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql | 1 + .../0_stateless/03088_analyzer_ambiguous_column_multi_call.sql | 1 + tests/queries/0_stateless/03089_analyzer_alias_replacement.sql | 1 + 65 files changed, 65 insertions(+) diff --git a/tests/queries/0_stateless/03033_cte_numbers_memory.sql b/tests/queries/0_stateless/03033_cte_numbers_memory.sql index 0e3ee9abd65..66b11cbfaa5 100644 --- a/tests/queries/0_stateless/03033_cte_numbers_memory.sql +++ b/tests/queries/0_stateless/03033_cte_numbers_memory.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/61238 +SET allow_experimental_analyzer=1; WITH (SELECT number FROM system.numbers LIMIT 1) as w1, diff --git a/tests/queries/0_stateless/03033_with_fill_interpolate.sql b/tests/queries/0_stateless/03033_with_fill_interpolate.sql index 816633af757..0ec0050a922 100644 --- a/tests/queries/0_stateless/03033_with_fill_interpolate.sql +++ b/tests/queries/0_stateless/03033_with_fill_interpolate.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/55794 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS 03033_example_table; CREATE TABLE 03033_example_table diff --git a/tests/queries/0_stateless/03034_normalized_ast.sql b/tests/queries/0_stateless/03034_normalized_ast.sql index dd109eb5113..385af4e2c34 100644 --- a/tests/queries/0_stateless/03034_normalized_ast.sql +++ b/tests/queries/0_stateless/03034_normalized_ast.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/49472 +SET allow_experimental_analyzer=1; SELECT concat(database, table) AS name, count() diff --git a/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql index 3a7b4890bf0..74463743b01 100644 --- a/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql +++ b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/44414 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS alias_bug; DROP TABLE IF EXISTS alias_bug_dist; CREATE TABLE alias_bug diff --git a/tests/queries/0_stateless/03036_with_numbers.sql b/tests/queries/0_stateless/03036_with_numbers.sql index 5e08bb6e065..3463ce826e2 100644 --- a/tests/queries/0_stateless/03036_with_numbers.sql +++ b/tests/queries/0_stateless/03036_with_numbers.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/13843 +SET allow_experimental_analyzer=1; WITH 10 AS n SELECT * FROM numbers(n); diff --git a/tests/queries/0_stateless/03037_union_view.sql b/tests/queries/0_stateless/03037_union_view.sql index fb8aa7df954..3ea81b829ba 100644 --- a/tests/queries/0_stateless/03037_union_view.sql +++ b/tests/queries/0_stateless/03037_union_view.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/55803 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS broken_table; DROP TABLE IF EXISTS broken_view; diff --git a/tests/queries/0_stateless/03038_ambiguous_column.sql b/tests/queries/0_stateless/03038_ambiguous_column.sql index 69c8e52d734..9df3cd9bc9b 100644 --- a/tests/queries/0_stateless/03038_ambiguous_column.sql +++ b/tests/queries/0_stateless/03038_ambiguous_column.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/48308 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS 03038_table; CREATE TABLE 03038_table diff --git a/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql b/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql index ca3bb521eba..640d217d2f9 100644 --- a/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql +++ b/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/45535 +SET allow_experimental_analyzer=1; SELECT *, diff --git a/tests/queries/0_stateless/03040_alias_column_join.sql b/tests/queries/0_stateless/03040_alias_column_join.sql index f4ea2e5914d..54f579c0feb 100644 --- a/tests/queries/0_stateless/03040_alias_column_join.sql +++ b/tests/queries/0_stateless/03040_alias_column_join.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/44365 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS 03040_test; CREATE TABLE 03040_test diff --git a/tests/queries/0_stateless/03040_array_sum_and_join.sql b/tests/queries/0_stateless/03040_array_sum_and_join.sql index 0084f0e4c7b..9aeddc9f765 100644 --- a/tests/queries/0_stateless/03040_array_sum_and_join.sql +++ b/tests/queries/0_stateless/03040_array_sum_and_join.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer=1; select t.1 as cnt, t.2 as name, diff --git a/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql b/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql index 462e63b121b..7906e65f8b8 100644 --- a/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql +++ b/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer=1; CREATE TABLE IF NOT EXISTS first engine = MergeTree PARTITION BY (inn, toYYYYMM(received)) ORDER BY (inn, sessionId) AS SELECT now() AS received, '123456789' AS inn, '42' AS sessionId; diff --git a/tests/queries/0_stateless/03041_select_with_query_result.sql b/tests/queries/0_stateless/03041_select_with_query_result.sql index 3edf51d635e..061223b43e1 100644 --- a/tests/queries/0_stateless/03041_select_with_query_result.sql +++ b/tests/queries/0_stateless/03041_select_with_query_result.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/44153 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS parent; DROP TABLE IF EXISTS join_table_1; DROP TABLE IF EXISTS join_table_2; diff --git a/tests/queries/0_stateless/03042_analyzer_alias_join.sql b/tests/queries/0_stateless/03042_analyzer_alias_join.sql index f3341fd314a..dac3b6a4983 100644 --- a/tests/queries/0_stateless/03042_analyzer_alias_join.sql +++ b/tests/queries/0_stateless/03042_analyzer_alias_join.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/14978 +SET allow_experimental_analyzer=1; CREATE TABLE test1(id UInt64, t1value UInt64) ENGINE=MergeTree ORDER BY tuple(); CREATE TABLE test2(id UInt64, t2value String) ENGINE=MergeTree ORDER BY tuple(); diff --git a/tests/queries/0_stateless/03042_not_found_column_c1.sql b/tests/queries/0_stateless/03042_not_found_column_c1.sql index 8ce7dcd9d4f..b4dce2af489 100644 --- a/tests/queries/0_stateless/03042_not_found_column_c1.sql +++ b/tests/queries/0_stateless/03042_not_found_column_c1.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/42399 +SET allow_experimental_analyzer=1; CREATE TABLE IF NOT EXISTS t0 (c0 Int32) ENGINE = Memory() ; CREATE TABLE t1 (c0 Int32, c1 Int32, c2 Int32) ENGINE = Memory() ; diff --git a/tests/queries/0_stateless/03043_group_array_result_is_expected.sql b/tests/queries/0_stateless/03043_group_array_result_is_expected.sql index df77ca66647..5311927ae3c 100644 --- a/tests/queries/0_stateless/03043_group_array_result_is_expected.sql +++ b/tests/queries/0_stateless/03043_group_array_result_is_expected.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/27115 +SET allow_experimental_analyzer=1; drop table if exists fill_ex; create table fill_ex ( diff --git a/tests/queries/0_stateless/03044_analyzer_alias_join.sql b/tests/queries/0_stateless/03044_analyzer_alias_join.sql index 5202b57a7b1..3ab8edb005f 100644 --- a/tests/queries/0_stateless/03044_analyzer_alias_join.sql +++ b/tests/queries/0_stateless/03044_analyzer_alias_join.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/17319 +SET allow_experimental_analyzer=1; CREATE TEMPORARY TABLE hits (date Date, data Float64) engine=Memory(); SELECT diff --git a/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql b/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql index f3ec80b8a94..0cf05763202 100644 --- a/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql +++ b/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql @@ -1,2 +1,3 @@ -- https://github.com/ClickHouse/ClickHouse/issues/11813 +SET allow_experimental_analyzer=1; select 1 from (select 1 x) l join (select 1 y, [1] a) r on l.x = r.y array join r.a; diff --git a/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql b/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql index a0546f57736..ee8756b9460 100644 --- a/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql +++ b/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/13210 +SET allow_experimental_analyzer=1; CREATE TABLE test_a_table ( name String, a_col String diff --git a/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql b/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql index cadcbdc0ce5..d97dfc880b3 100644 --- a/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql +++ b/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/23053 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS repl_tbl; CREATE TEMPORARY TABLE repl_tbl diff --git a/tests/queries/0_stateless/03046_column_in_block_array_join.sql b/tests/queries/0_stateless/03046_column_in_block_array_join.sql index 9a2bb19d81e..c6b4613af3f 100644 --- a/tests/queries/0_stateless/03046_column_in_block_array_join.sql +++ b/tests/queries/0_stateless/03046_column_in_block_array_join.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/37729 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS nested_test; DROP TABLE IF EXISTS join_test; diff --git a/tests/queries/0_stateless/03047_analyzer_alias_join.sql b/tests/queries/0_stateless/03047_analyzer_alias_join.sql index ef8c067bb72..7d44c92b6f1 100644 --- a/tests/queries/0_stateless/03047_analyzer_alias_join.sql +++ b/tests/queries/0_stateless/03047_analyzer_alias_join.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer=1; SELECT 1 AS value, * diff --git a/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql b/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql index e1363ea4dda..cfaf1df44bd 100644 --- a/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql +++ b/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql @@ -1,3 +1,4 @@ -- https://github.com/ClickHouse/ClickHouse/issues/32639 +SET allow_experimental_analyzer=1; SELECT 0 AND id ? 1 : 2 AS a, sum(id) FROM (SELECT 1 AS id) GROUP BY a; diff --git a/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql b/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql index 25f88050eb1..42fd581e142 100644 --- a/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql +++ b/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/41964 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS ab_12_aaa; DROP TABLE IF EXISTS ab_12_bbb; diff --git a/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql b/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql index 67df48e0cad..d25babe6788 100644 --- a/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql +++ b/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/7520 +SET allow_experimental_analyzer=1; CREATE TABLE test (`a` UInt32, `b` UInt32) ENGINE = Memory; INSERT INTO test VALUES (1,2), (1,3), (2,4); diff --git a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql index 2f12799addb..a1c858a329c 100644 --- a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql +++ b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/54317 +SET allow_experimental_analyzer=1; DROP DATABASE IF EXISTS 03049_database; DROP TABLE IF EXISTS 03049_database.l; DROP TABLE IF EXISTS 03049_database.r; diff --git a/tests/queries/0_stateless/03050_select_one_one_one.sql b/tests/queries/0_stateless/03050_select_one_one_one.sql index eee973fe936..28a55e0c471 100644 --- a/tests/queries/0_stateless/03050_select_one_one_one.sql +++ b/tests/queries/0_stateless/03050_select_one_one_one.sql @@ -1,3 +1,4 @@ -- https://github.com/ClickHouse/ClickHouse/issues/36973 +SET allow_experimental_analyzer=1; SELECT 1, 1, 1; SELECT * FROM (SELECT 1, 1, 1); diff --git a/tests/queries/0_stateless/03051_many_ctes.sql b/tests/queries/0_stateless/03051_many_ctes.sql index 412a1e6b544..d4e613bd279 100644 --- a/tests/queries/0_stateless/03051_many_ctes.sql +++ b/tests/queries/0_stateless/03051_many_ctes.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/40955 +SET allow_experimental_analyzer=1; WITH toInt64(2) AS new_x SELECT new_x AS x FROM (SELECT 1 AS x) t; WITH toInt64(2) AS new_x SELECT * replace(new_x as x) FROM (SELECT 1 AS x) t; SELECT 2 AS x FROM (SELECT 1 AS x) t; diff --git a/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql b/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql index e76108c7842..24e9ab0f36e 100644 --- a/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql +++ b/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/pull/40065 +SET allow_experimental_analyzer=1; SELECT ( diff --git a/tests/queries/0_stateless/03053_analyzer_join_alias.sql b/tests/queries/0_stateless/03053_analyzer_join_alias.sql index 7e11cc7c810..ef51ec73026 100644 --- a/tests/queries/0_stateless/03053_analyzer_join_alias.sql +++ b/tests/queries/0_stateless/03053_analyzer_join_alias.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/23104 +SET allow_experimental_analyzer=1; DROP DATABASE IF EXISTS test_03053; CREATE DATABASE test_03053; diff --git a/tests/queries/0_stateless/03054_analyzer_join_alias.sql b/tests/queries/0_stateless/03054_analyzer_join_alias.sql index 0bf93258aa6..e124aa33a9b 100644 --- a/tests/queries/0_stateless/03054_analyzer_join_alias.sql +++ b/tests/queries/0_stateless/03054_analyzer_join_alias.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/21584 +SET allow_experimental_analyzer=1; SELECT count() FROM ( diff --git a/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql b/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql index 071d8f8e1c8..25b6dcb3564 100644 --- a/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql +++ b/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/23344 +SET allow_experimental_analyzer=1; SELECT logTrace(repeat('Hello', 100)), ignore(*) FROM ( SELECT ignore((SELECT groupArrayState(([number], [number])) FROM numbers(19000))) diff --git a/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql b/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql index b86ae97c8bf..de471c1a091 100644 --- a/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql +++ b/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/22627 +SET allow_experimental_analyzer=1; WITH x AS ( diff --git a/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql b/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql index 13852471dca..2217af327fa 100644 --- a/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql +++ b/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/10276 +SET allow_experimental_analyzer=1; SELECT sum(x.n) as n, sum(z.n) as n2 diff --git a/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql b/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql index 47df6e76a38..3cce77f0240 100644 --- a/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql +++ b/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/4567 +SET allow_experimental_analyzer=1; DROP TABLE IF EXISTS fact; DROP TABLE IF EXISTS animals; DROP TABLE IF EXISTS colors; diff --git a/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql b/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql index 76150335f7e..27782462075 100644 --- a/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql +++ b/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/17710 +SET allow_experimental_analyzer=1; CREATE TABLE id_val(id UInt32, val UInt32) ENGINE = Memory; CREATE TABLE id_val_join0(id UInt32, val UInt8) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls = 0; CREATE TABLE id_val_join1(id UInt32, val UInt8) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls = 1; diff --git a/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql b/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql index ba0257d7b3b..f8cd8690ee5 100644 --- a/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql +++ b/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/11068 +SET allow_experimental_analyzer=1; create table vt(datetime_value DateTime, value Float64) Engine=Memory; create view computed_datum_hours as diff --git a/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql b/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql index e223909a5a8..6fee6d1f73d 100644 --- a/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql +++ b/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/24395 +SET allow_experimental_analyzer=1; CREATE TABLE xxxx_yyy (key UInt32, key_b ALIAS key) ENGINE=MergeTree() ORDER BY key; INSERT INTO xxxx_yyy SELECT number FROM numbers(10); diff --git a/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql b/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql index 6c24ef6f66d..9748175e4d4 100644 --- a/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql +++ b/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/23416 +SET allow_experimental_analyzer=1; create table test (TOPIC String, PARTITION UInt64, OFFSET UInt64, ID UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_03062', 'r2') ORDER BY (TOPIC, PARTITION, OFFSET); create table test_join (TOPIC String, PARTITION UInt64, OFFSET UInt64) ENGINE = Join(ANY, LEFT, `TOPIC`, `PARTITION`) SETTINGS join_any_take_last_row = 1; diff --git a/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql b/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql index c2c29b688cd..7eab1fa846a 100644 --- a/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql +++ b/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/23162 +SET allow_experimental_analyzer=1; CREATE TABLE t1 ( k Int64, x Int64) ENGINE = Memory; CREATE TABLE t2( x Int64 ) ENGINE = Memory; diff --git a/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql b/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql index ef8aca2fefa..59ebb9d9af3 100644 --- a/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql +++ b/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/25655 +SET allow_experimental_analyzer=1; SELECT sum(t.b) / 1 a, sum(t.a) diff --git a/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql b/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql index c270a0f4504..7e6befe181e 100644 --- a/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql +++ b/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql @@ -1,2 +1,3 @@ -- https://github.com/ClickHouse/ClickHouse/issues/11757 +SET allow_experimental_analyzer=1; select * from (select [1, 2] a) aa cross join (select [3, 4] b) bb array join aa.a, bb.b; diff --git a/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql b/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql index 338eb30e6ff..8983be242c3 100644 --- a/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql +++ b/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer=1; WITH 0 AS test SELECT * FROM diff --git a/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql b/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql index 7d1264a6116..052a9eaf734 100644 --- a/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql +++ b/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer=1; with d as (select 'key'::Varchar(255) c, 'x'::Varchar(255) s) SELECT r1, c as r2 FROM ( diff --git a/tests/queries/0_stateless/03068_analyzer_distributed_join.sql b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql index 714a64a01b6..82f58e9a750 100644 --- a/tests/queries/0_stateless/03068_analyzer_distributed_join.sql +++ b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/6571 +SET allow_experimental_analyzer=1; CREATE TABLE LINEITEM_shard ON CLUSTER test_shard_localhost ( L_ORDERKEY UInt64, diff --git a/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql b/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql index 84ad0b4e199..09d2985fe60 100644 --- a/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql +++ b/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/4432 +SET allow_experimental_analyzer=1; WITH [1, 2] AS zz SELECT x FROM system.one diff --git a/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql b/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql index 672c4f53e5f..7aadab2ca73 100644 --- a/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql +++ b/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/8259 +SET allow_experimental_analyzer=1; with (select 25) as something select *, something diff --git a/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql b/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql index af81e3c2819..e2eb758d649 100644 --- a/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql +++ b/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/9233 +SET allow_experimental_analyzer=1; SELECT * FROM ( diff --git a/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql b/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql index 68ff81413b7..e2846033913 100644 --- a/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql +++ b/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql @@ -1,2 +1,3 @@ -- https://github.com/ClickHouse/ClickHouse/issues/14699 +SET allow_experimental_analyzer=1; select * from (select number from numbers(1)) where not ignore(*); diff --git a/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql b/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql index 0e7d2eb95c7..5599324c62b 100644 --- a/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql +++ b/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/27068 +SET allow_experimental_analyzer=1; CREATE TABLE test ( id String, create_time DateTime ) ENGINE = MergeTree ORDER BY id; insert into test values(1,'1970-02-01 00:00:00'); diff --git a/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql b/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql index 6f970421788..4df5f6f48e6 100644 --- a/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql +++ b/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/28687 +SET allow_experimental_analyzer=1; create view alias (dummy int, n alias dummy) as select * from system.one; select n from alias; diff --git a/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql b/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql index 897b189b133..416815e761b 100644 --- a/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql +++ b/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/28777 +SET allow_experimental_analyzer=1; SELECT sum(q0.a2) AS a1, sum(q0.a1) AS a9 diff --git a/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql b/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql index 8b8b76a5be1..7ac9fe6b446 100644 --- a/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql +++ b/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/29734 +SET allow_experimental_analyzer=1; SELECT * FROM ( diff --git a/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql b/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql index 3d558bdd602..5a181023c57 100644 --- a/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql +++ b/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/33825 +SET allow_experimental_analyzer=1; CREATE TABLE t1 (i Int64, j Int64) ENGINE = Memory; INSERT INTO t1 SELECT number, number FROM system.numbers LIMIT 10; SELECT diff --git a/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql b/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql index ded6bfbe4e3..d91a9ed106d 100644 --- a/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql +++ b/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/33825 +SET allow_experimental_analyzer=1; CREATE TABLE t2 (first_column Int64, second_column Int64) ENGINE = Memory; INSERT INTO t2 SELECT number, number FROM system.numbers LIMIT 10; diff --git a/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql b/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql index e6bcad34506..955d3b49a00 100644 --- a/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql +++ b/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql @@ -1,3 +1,4 @@ +SET allow_experimental_analyzer=1; CREATE TABLE testdata (`1` String) ENGINE=MergeTree ORDER BY tuple(); INSERT INTO testdata VALUES ('testdata'); diff --git a/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql b/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql index 1fe19cdad2a..01ab868f9ea 100644 --- a/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql +++ b/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/35652 +SET allow_experimental_analyzer=1; CREATE TABLE test ( id UInt64 ) diff --git a/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql b/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql index ae8e40f6d56..4985d3abfb6 100644 --- a/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql +++ b/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/29838 +SET allow_experimental_analyzer=1; CREATE TABLE first_table_lr ( diff --git a/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql b/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql index 81dbbb3b62d..e6a540dc5df 100644 --- a/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql +++ b/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/36189 +SET allow_experimental_analyzer=1; CREATE TABLE test ( `dt` Date, diff --git a/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql b/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql index 93702fee550..8f17248ed0d 100644 --- a/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql +++ b/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/39634 +SET allow_experimental_analyzer=1; CREATE TABLE test1 ( `pk` String, diff --git a/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql b/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql index caf65823532..8337c0ce987 100644 --- a/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql +++ b/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/47432 +SET allow_experimental_analyzer=1; create or replace table t1 engine = MergeTree() order by tuple() diff --git a/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql b/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql index f4eaa5d9710..fd67194b08b 100644 --- a/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql +++ b/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/54910 +SET allow_experimental_analyzer=1; SELECT toTypeName(stat_standard_id) AS stat_standard_id_1, count(1) AS value FROM ( SELECT 'string value' AS stat_standard_id ) GROUP BY stat_standard_id_1 LIMIT 1 diff --git a/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql index 55a60873a5a..ac03019de20 100644 --- a/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql +++ b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/57321 +SET allow_experimental_analyzer=1; SELECT ver, max(ver) OVER () AS ver_max diff --git a/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql b/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql index 98aca76fe49..6546e50c99e 100644 --- a/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql +++ b/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/59154 +SET allow_experimental_analyzer=1; SELECT * FROM ( diff --git a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql index 84afdb295c2..09425d2e503 100644 --- a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql +++ b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/61014 +SET allow_experimental_analyzer=1; create database test_03088; create table test_03088.a (i int) engine = Log(); diff --git a/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql b/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql index 00a3795eab8..069da5fdd65 100644 --- a/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql +++ b/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql @@ -1,4 +1,5 @@ -- https://github.com/ClickHouse/ClickHouse/issues/61950 +SET allow_experimental_analyzer=1; with dummy + 1 as dummy select dummy from system.one; From cd1e96a8a10c026214a07bc6bf251c008236fa3c Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Thu, 4 Apr 2024 16:22:17 +0200 Subject: [PATCH 0992/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/55647 --- ...analyzer_multiple_using_statements.reference | 1 + ...03090_analyzer_multiple_using_statements.sql | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/queries/0_stateless/03090_analyzer_multiple_using_statements.reference create mode 100644 tests/queries/0_stateless/03090_analyzer_multiple_using_statements.sql diff --git a/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.reference b/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.sql b/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.sql new file mode 100644 index 00000000000..c35f33782ff --- /dev/null +++ b/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.sql @@ -0,0 +1,17 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/55647 +SET allow_experimental_analyzer=1; + +SELECT +* +FROM ( + SELECT * + FROM system.one +) a +JOIN ( + SELECT * + FROM system.one +) b USING dummy +JOIN ( + SELECT * + FROM system.one +) c USING dummy From 32d124e9033bd3785eb1b301f908bf8733ff433d Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Thu, 4 Apr 2024 16:30:42 +0200 Subject: [PATCH 0993/1165] Close: https://github.com/ClickHouse/ClickHouse/issues/61947 --- ...able_name_in_different_databases.reference | 4 +++ ...same_table_name_in_different_databases.sql | 28 +++++++++++++++++++ ...able_name_in_different_databases.reference | 1 + ...same_table_name_in_different_databases.sql | 18 ++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.reference create mode 100644 tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql create mode 100644 tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.reference create mode 100644 tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql diff --git a/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.reference b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.reference new file mode 100644 index 00000000000..ce45f6636b2 --- /dev/null +++ b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.reference @@ -0,0 +1,4 @@ +1 0 + +using asterisk 1 0 +using field name 1 0 diff --git a/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql new file mode 100644 index 00000000000..2185b5f450a --- /dev/null +++ b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql @@ -0,0 +1,28 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/61947 +SET allow_experimental_analyzer=1; + +DROP DATABASE IF EXISTS d1; +DROP DATABASE IF EXISTS d2; + +CREATE DATABASE d1; +CREATE DATABASE d2; +CREATE TABLE d1.`1-1` (field Int8) ENGINE = Memory; +CREATE TABLE d2.`1-1` (field Int8) ENGINE = Memory; +CREATE TABLE d2.`2-1` (field Int8) ENGINE = Memory; + +INSERT INTO d1.`1-1` VALUES (1); + +SELECT * +FROM d1.`1-1` +LEFT JOIN d2.`1-1` ON d1.`1-1`.field = d2.`1-1`.field; + +SELECT ''; + +SELECT 'using asterisk', d1.`1-1`.*, d2.`1-1`.* +FROM d1.`1-1` +LEFT JOIN d2.`1-1` USING field +UNION ALL +SELECT 'using field name', d1.`1-1`.field, d2.`1-1`.field +FROM d1.`1-1` +LEFT JOIN d2.`1-1` USING field +ORDER BY *; diff --git a/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.reference b/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql b/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql new file mode 100644 index 00000000000..03ad9c97d94 --- /dev/null +++ b/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql @@ -0,0 +1,18 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/61947 +SET allow_experimental_analyzer=1; + +DROP DATABASE IF EXISTS d3; +DROP DATABASE IF EXISTS d4; + +CREATE DATABASE d3; +CREATE DATABASE d4; +CREATE TABLE d3.`1-1` (field Int8) ENGINE = Memory; +CREATE TABLE d4.`2-1` (field Int8) ENGINE = Memory; +CREATE TABLE d4.`3-1` (field Int8) ENGINE = Memory; + +INSERT INTO d3.`1-1` VALUES (1); + +SELECT d3.`1-1`.* +FROM d3.`1-1` +LEFT JOIN d4.`2-1` ON d3.`1-1`.field = d4.`2-1`.field +LEFT JOIN d4.`3-1` ON d4.`2-1`.field = d4.`3-1`.field; From 34a21199451d99cef8190a80e9ca498b0bfde175 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 4 Apr 2024 16:32:29 +0200 Subject: [PATCH 0994/1165] Public visibility --- contrib/nuraft-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/nuraft-cmake/CMakeLists.txt b/contrib/nuraft-cmake/CMakeLists.txt index 140fb11c6d0..a57b7ac6225 100644 --- a/contrib/nuraft-cmake/CMakeLists.txt +++ b/contrib/nuraft-cmake/CMakeLists.txt @@ -52,7 +52,7 @@ else() endif() target_link_libraries (_nuraft PRIVATE clickhouse_common_io) -target_compile_definitions(_nuraft PRIVATE USE_CLICKHOUSE_THREADS=1) +target_compile_definitions(_nuraft PUBLIC USE_CLICKHOUSE_THREADS=1) MESSAGE(STATUS "Will use clickhouse threads for NuRaft") target_include_directories (_nuraft SYSTEM PRIVATE "${LIBRARY_DIR}/include/libnuraft") From d9e7c0a662b609ae747a729e19e992b8591f7cd8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 4 Apr 2024 16:36:12 +0200 Subject: [PATCH 0995/1165] Add comment --- contrib/nuraft-cmake/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/nuraft-cmake/CMakeLists.txt b/contrib/nuraft-cmake/CMakeLists.txt index a57b7ac6225..736e91e359d 100644 --- a/contrib/nuraft-cmake/CMakeLists.txt +++ b/contrib/nuraft-cmake/CMakeLists.txt @@ -52,6 +52,8 @@ else() endif() target_link_libraries (_nuraft PRIVATE clickhouse_common_io) +# We must have it PUBLIC here because some headers which depend on it directly +# included in clickhouse target_compile_definitions(_nuraft PUBLIC USE_CLICKHOUSE_THREADS=1) MESSAGE(STATUS "Will use clickhouse threads for NuRaft") From 58e6bd82cff7c18e19ce52ea97c993456aba60df Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Thu, 4 Apr 2024 16:37:49 +0200 Subject: [PATCH 0996/1165] Add missing ORDER BY --- .../0_stateless/03086_analyzer_window_func_part_of_group_by.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql index ac03019de20..31747328d1f 100644 --- a/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql +++ b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql @@ -10,3 +10,4 @@ FROM SELECT 2 AS ver ) GROUP BY ver +ORDER BY ver; From 85d98e1a421cecb8be9051331f8fdd46bfd557ea Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 4 Apr 2024 16:51:46 +0200 Subject: [PATCH 0997/1165] Modernize code a little --- src/Common/QueryProfiler.cpp | 16 ++-------------- src/Common/ThreadStatus.cpp | 2 +- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index b616b795405..d9ff3a86e4a 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -210,23 +210,13 @@ void Timer::cleanup() #endif template -QueryProfilerBase::QueryProfilerBase(UInt64 thread_id, int clock_type, UInt32 period, int pause_signal_) +QueryProfilerBase::QueryProfilerBase([[maybe_unused]] UInt64 thread_id, [[maybe_unused]] int clock_type, [[maybe_unused]] UInt32 period, [[maybe_unused]] int pause_signal_) : log(getLogger("QueryProfiler")) , pause_signal(pause_signal_) { #if defined(SANITIZER) - UNUSED(thread_id); - UNUSED(clock_type); - UNUSED(period); - UNUSED(pause_signal); - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QueryProfiler disabled because they cannot work under sanitizers"); #elif defined(__APPLE__) - UNUSED(thread_id); - UNUSED(clock_type); - UNUSED(period); - UNUSED(pause_signal); - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QueryProfiler cannot work on OSX"); #else /// Sanity check. @@ -262,13 +252,11 @@ QueryProfilerBase::QueryProfilerBase(UInt64 thread_id, int clock_t template -void QueryProfilerBase::setPeriod(UInt32 period_) +void QueryProfilerBase::setPeriod([[maybe_unused]] UInt32 period_) { #if defined(SANITIZER) - UNUSED(period); throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QueryProfiler disabled because they cannot work under sanitizers"); #elif defined(__APPLE__) - UNUSED(period); throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QueryProfiler cannot work on OSX"); #else timer.set(period_); diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index a3a7257b472..d4a40360a3b 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -124,7 +124,7 @@ ThreadStatus::ThreadStatus(bool check_current_thread_on_destruction_) #endif } -void ThreadStatus::initGlobalProfiler(UInt64 global_profiler_real_time_period, UInt64 global_profiler_cpu_time_period) +void ThreadStatus::initGlobalProfiler([[maybe_unused]] UInt64 global_profiler_real_time_period, [[maybe_unused]] UInt64 global_profiler_cpu_time_period) { #if !defined(SANITIZER) && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD) && !defined(__APPLE__) try From ff054b5ab304a866e7fb95628082f582216cb67a Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 4 Apr 2024 17:14:55 +0200 Subject: [PATCH 0998/1165] With better name --- contrib/NuRaft | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index 717657cd94d..cb5dc3c906e 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 717657cd94da01e86733e58f8d3f0ca0d8748712 +Subproject commit cb5dc3c906e80f253e9ce9535807caef827cc2e0 From 51aff6d63c4c55a8d18470103a24f004fde0ee67 Mon Sep 17 00:00:00 2001 From: Misz606 <113922942+Misz606@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:09:05 -0400 Subject: [PATCH 0999/1165] Update README.md Localy should be spelled locally. --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index a1fc0f530f2..7df5da8d078 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1 +1 @@ -Find CI documents and instructions on running CI checks localy [here](https://clickhouse.com/docs/en/development/continuous-integration). \ No newline at end of file +Find CI documents and instructions on running CI checks locally [here](https://clickhouse.com/docs/en/development/continuous-integration). From eede7c3acf5e1fcc113349b4014aaf5b981ce285 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 4 Apr 2024 19:01:31 +0200 Subject: [PATCH 1000/1165] Try fix stress test --- src/Interpreters/Cache/EvictionCandidates.cpp | 2 ++ src/Interpreters/Cache/FileCache.cpp | 31 ++++++++++++++----- src/Interpreters/Cache/Metadata.cpp | 1 - .../Cache/SLRUFileCachePriority.cpp | 10 +++++- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 5fa2b337e64..da9e3efbffe 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -63,6 +63,8 @@ void EvictionCandidates::add( void EvictionCandidates::removeQueueEntries(const CachePriorityGuard::Lock & lock) { + auto log = getLogger("EvictionCandidates"); + LOG_TEST(log, "Will remove {} eviction candidates", size()); for (const auto & [key, key_candidates] : candidates) { for (const auto & candidate : key_candidates.candidates) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 90671629e64..5e97c0110f4 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1383,6 +1383,7 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, || new_settings.max_elements != actual_settings.max_elements) { std::optional eviction_candidates; + bool modified_size_limits = false; { cache_is_being_resized.store(true, std::memory_order_relaxed); SCOPE_EXIT({ @@ -1397,8 +1398,21 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, eviction_candidates->removeQueueEntries(cache_lock); - main_priority->modifySizeLimits( - new_settings.max_size, new_settings.max_elements, new_settings.slru_size_ratio, cache_lock); + modified_size_limits = main_priority->getSize(cache_lock) <= new_settings.max_size + && main_priority->getElementsCount(cache_lock) <= new_settings.max_elements; + + if (modified_size_limits) + { + main_priority->modifySizeLimits( + new_settings.max_size, new_settings.max_elements, new_settings.slru_size_ratio, cache_lock); + } + else + { + LOG_WARNING(log, "Unable to modify size limit from {} to {}, " + "elements limit from {} to {}", + actual_settings.max_size, new_settings.max_size, + actual_settings.max_elements, new_settings.max_elements); + } } try @@ -1412,12 +1426,15 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, throw; } - LOG_INFO(log, "Changed max_size from {} to {}, max_elements from {} to {}", - actual_settings.max_size, new_settings.max_size, - actual_settings.max_elements, new_settings.max_elements); + if (modified_size_limits) + { + LOG_INFO(log, "Changed max_size from {} to {}, max_elements from {} to {}", + actual_settings.max_size, new_settings.max_size, + actual_settings.max_elements, new_settings.max_elements); - actual_settings.max_size = new_settings.max_size; - actual_settings.max_elements = new_settings.max_elements; + actual_settings.max_size = new_settings.max_size; + actual_settings.max_elements = new_settings.max_elements; + } } if (new_settings.max_file_segment_size != actual_settings.max_file_segment_size) diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 26611f02379..631c1aa2ae6 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -941,7 +941,6 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl( file_segment->detach(segment_lock, *this); - // if (!remove_only_metadata) { try { diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 79ca489cea2..ff583c440c8 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -262,6 +262,9 @@ EvictionCandidates SLRUFileCachePriority::collectCandidatesForEviction( auto res = probationary_queue.collectCandidatesForEviction( desired_probationary_size, desired_probationary_elements_num, max_candidates_to_evict, stat, lock); + LOG_TEST(log, "Collected {} eviction candidates from probationary queue (size: {})", + res.size(), stat.total_stat.releasable_size); + chassert(!max_candidates_to_evict || res.size() <= max_candidates_to_evict); chassert(res.size() == stat.total_stat.releasable_count); @@ -271,10 +274,15 @@ EvictionCandidates SLRUFileCachePriority::collectCandidatesForEviction( const auto desired_protected_size = getRatio(max_size, size_ratio); const auto desired_protected_elements_num = getRatio(max_elements, size_ratio); + FileCacheReserveStat protected_stat; auto res_add = protected_queue.collectCandidatesForEviction( desired_protected_size, desired_protected_elements_num, - max_candidates_to_evict ? max_candidates_to_evict - res.size() : 0, stat, lock); + max_candidates_to_evict ? max_candidates_to_evict - res.size() : 0, protected_stat, lock); + LOG_TEST(log, "Collected {} eviction candidates from protected queue (size: {})", + res_add.size(), protected_stat.total_stat.releasable_size); + + stat += protected_stat; res.insert(std::move(res_add), lock); return res; } From 7ef5e25a2609380d039306fa540b57eac388d7da Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 4 Apr 2024 19:16:38 +0200 Subject: [PATCH 1001/1165] empty commit From 0fccaafda45ccf85eeeacc6c8cd6f417fed91e73 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Apr 2024 20:06:40 +0200 Subject: [PATCH 1002/1165] Fix tests --- docker/test/stateless/run.sh | 2 ++ tests/ci/integration_tests_runner.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index b9ed0561a48..d7088aa05fb 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -16,6 +16,8 @@ ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone dpkg -i package_folder/clickhouse-common-static_*.deb dpkg -i package_folder/clickhouse-common-static-dbg_*.deb +dpkg -i package_folder/clickhouse-odbc-bridge_*.deb +dpkg -i package_folder/clickhouse-library-bridge_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py index f50124500cc..90e2b08386f 100755 --- a/tests/ci/integration_tests_runner.py +++ b/tests/ci/integration_tests_runner.py @@ -342,6 +342,8 @@ class ClickhouseIntegrationTestsRunner: "clickhouse-common-static_", "clickhouse-server_", "clickhouse-client", + "clickhouse-odbc-bridge_", + "clickhouse-library-bridge_", "clickhouse-common-static-dbg_", ): # order matters logging.info("Installing package %s", package) From 3e1e4348333fb026c17420be34871f1638bcf2e0 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 4 Apr 2024 19:52:40 +0000 Subject: [PATCH 1003/1165] Fix data race on scalars in Context --- src/Interpreters/Context.cpp | 11 +- .../03033_scalars_context_data_race.reference | 1 + .../03033_scalars_context_data_race.sql | 104 ++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03033_scalars_context_data_race.reference create mode 100644 tests/queries/0_stateless/03033_scalars_context_data_race.sql diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 65fcd51529b..0f28e5d8771 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1552,12 +1552,15 @@ ClassifierPtr Context::getWorkloadClassifier() const const Scalars & Context::getScalars() const { + std::lock_guard lock(mutex); return scalars; } const Block & Context::getScalar(const String & name) const { + std::lock_guard lock(mutex); + auto it = scalars.find(name); if (scalars.end() == it) { @@ -1570,6 +1573,7 @@ const Block & Context::getScalar(const String & name) const const Block * Context::tryGetSpecialScalar(const String & name) const { + std::lock_guard lock(mutex); auto it = special_scalars.find(name); if (special_scalars.end() == it) return nullptr; @@ -1653,7 +1657,8 @@ void Context::addScalar(const String & name, const Block & block) if (isGlobalContext()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot have scalars"); - scalars[name] = block; + std::lock_guard lock(mutex); + scalars.emplace(name, block); } @@ -1662,7 +1667,8 @@ void Context::addSpecialScalar(const String & name, const Block & block) if (isGlobalContext()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot have local scalars"); - special_scalars[name] = block; + std::lock_guard lock(mutex); + special_scalars.emplace(name, block); } @@ -1671,6 +1677,7 @@ bool Context::hasScalar(const String & name) const if (isGlobalContext()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot have scalars"); + std::lock_guard lock(mutex); return scalars.contains(name); } diff --git a/tests/queries/0_stateless/03033_scalars_context_data_race.reference b/tests/queries/0_stateless/03033_scalars_context_data_race.reference new file mode 100644 index 00000000000..f96ac067218 --- /dev/null +++ b/tests/queries/0_stateless/03033_scalars_context_data_race.reference @@ -0,0 +1 @@ +105 diff --git a/tests/queries/0_stateless/03033_scalars_context_data_race.sql b/tests/queries/0_stateless/03033_scalars_context_data_race.sql new file mode 100644 index 00000000000..8c72bb53c72 --- /dev/null +++ b/tests/queries/0_stateless/03033_scalars_context_data_race.sql @@ -0,0 +1,104 @@ +DROP TABLE IF EXISTS test; +DROP TABLE IF EXISTS test_tmp; +DROP TABLE IF EXISTS dst; +DROP TABLE IF EXISTS view; + +CREATE TABLE test +( + `address` FixedString(20), + `deployer` FixedString(20), + `block_number` UInt256, + `block_hash` FixedString(32), + `block_timestamp` DateTime('UTC'), + `insertion_time` DateTime('UTC') +) +ENGINE = MergeTree +ORDER BY address +SETTINGS index_granularity = 8192; + +CREATE TABLE test_tmp as test; + +CREATE TABLE dst +( + `block_timestamp` AggregateFunction(max, Nullable(DateTime('UTC'))), + `block_hash` AggregateFunction(argMax, Nullable(FixedString(32)), DateTime('UTC')), + `block_number` AggregateFunction(argMax, Nullable(UInt256), DateTime('UTC')), + `deployer` AggregateFunction(argMax, Nullable(FixedString(20)), DateTime('UTC')), + `address` FixedString(20), + `name` AggregateFunction(argMax, Nullable(String), DateTime('UTC')), + `symbol` AggregateFunction(argMax, Nullable(String), DateTime('UTC')), + `decimals` AggregateFunction(argMax, Nullable(UInt8), DateTime('UTC')), + `is_proxy` AggregateFunction(argMax, Nullable(Bool), DateTime('UTC')), + `blacklist_flags` AggregateFunction(argMax, Array(Nullable(String)), DateTime('UTC')), + `whitelist_flags` AggregateFunction(argMax, Array(Nullable(String)), DateTime('UTC')), + `detected_standards` AggregateFunction(argMax, Array(Nullable(String)), DateTime('UTC')), + `amended_type` AggregateFunction(argMax, Nullable(String), DateTime('UTC')), + `comment` AggregateFunction(argMax, Nullable(String), DateTime('UTC')), + `_sources` AggregateFunction(groupUniqArray, String), + `_updated_at` AggregateFunction(max, DateTime('UTC')), + `_active` AggregateFunction(argMax, Bool, DateTime('UTC')) +) +ENGINE = MergeTree +ORDER BY address +SETTINGS index_granularity = 8192; + +CREATE MATERIALIZED VIEW view TO dst +( + `block_timestamp` AggregateFunction(max, Nullable(DateTime('UTC'))), + `block_hash` AggregateFunction(argMax, Nullable(FixedString(32)), DateTime('UTC')), + `block_number` AggregateFunction(argMax, Nullable(UInt256), DateTime('UTC')), + `deployer` AggregateFunction(argMax, Nullable(FixedString(20)), DateTime('UTC')), + `address` FixedString(20), + `name` AggregateFunction(argMax, Nullable(String), DateTime('UTC')), + `symbol` AggregateFunction(argMax, Nullable(String), DateTime('UTC')), + `decimals` AggregateFunction(argMax, Nullable(UInt8), DateTime('UTC')), + `is_proxy` AggregateFunction(argMax, Nullable(Bool), DateTime('UTC')), + `blacklist_flags` AggregateFunction(argMax, Array(Nullable(String)), DateTime('UTC')), + `whitelist_flags` AggregateFunction(argMax, Array(Nullable(String)), DateTime('UTC')), + `detected_standards` AggregateFunction(argMax, Array(Nullable(String)), DateTime('UTC')), + `amended_type` AggregateFunction(argMax, Nullable(String), DateTime('UTC')), + `comment` AggregateFunction(argMax, Nullable(String), DateTime('UTC')), + `_sources` AggregateFunction(groupUniqArray, String), + `_updated_at` AggregateFunction(max, DateTime('UTC')), + `_active` AggregateFunction(argMax, Bool, DateTime('UTC')) +) AS +(WITH ( + SELECT toDateTime('1970-01-01 00:00:00') + ) AS default_timestamp +SELECT + maxState(CAST(block_timestamp, 'Nullable(DateTime(\'UTC\'))')) AS block_timestamp, + argMaxState(CAST(block_hash, 'Nullable(FixedString(32))'), insertion_time) AS block_hash, + argMaxState(CAST(block_number, 'Nullable(UInt256)'), insertion_time) AS block_number, + argMaxState(CAST(deployer, 'Nullable(FixedString(20))'), insertion_time) AS deployer, + address, + argMaxState(CAST(NULL, 'Nullable(String)'), CAST(default_timestamp, 'DateTime(\'UTC\')')) AS name, + argMaxState(CAST(NULL, 'Nullable(String)'), CAST(default_timestamp, 'DateTime(\'UTC\')')) AS symbol, + argMaxState(CAST(NULL, 'Nullable(UInt8)'), CAST(default_timestamp, 'DateTime(\'UTC\')')) AS decimals, + argMaxState(CAST(true, 'Nullable(Boolean)'), insertion_time) AS is_proxy, + argMaxState(CAST('[]', 'Array(Nullable(String))'), CAST(default_timestamp, 'DateTime(\'UTC\')')) AS blacklist_flags, + argMaxState(CAST('[]', 'Array(Nullable(String))'), CAST(default_timestamp, 'DateTime(\'UTC\')')) AS whitelist_flags, + argMaxState(CAST('[]', 'Array(Nullable(String))'), CAST(default_timestamp, 'DateTime(\'UTC\')')) AS detected_standards, + argMaxState(CAST(NULL, 'Nullable(String)'), CAST(default_timestamp, 'DateTime(\'UTC\')')) AS amended_type, + argMaxState(CAST(NULL, 'Nullable(String)'), CAST(default_timestamp, 'DateTime(\'UTC\')')) AS comment, + groupUniqArrayState('tokens_proxy_deployments') AS _sources, + maxState(insertion_time) AS _updated_at, + argMaxState(true, CAST(default_timestamp, 'DateTime(\'UTC\')')) AS _active +FROM test +WHERE insertion_time > toDateTime('2024-03-14 11:38:09') +GROUP BY address); + +set max_insert_threads=4; +insert into test_tmp select * from generateRandom() limit 24; +insert into test_tmp select * from generateRandom() limit 25; +insert into test_tmp select * from generateRandom() limit 26; +insert into test_tmp select * from generateRandom() limit 30; + +INSERT INTO test(address, deployer, block_number, block_hash, block_timestamp, insertion_time) SELECT * FROM test_tmp; + +select count() from test; + +DROP TABLE test; +DROP TABLE test_tmp; +DROP TABLE dst; +DROP TABLE view; + From d7fb851d172e2955ab81ea107ed58c0867a1929f Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 4 Apr 2024 19:53:58 +0000 Subject: [PATCH 1004/1165] better handling of errors from azure storage --- .../IO/ReadBufferFromAzureBlobStorage.cpp | 41 ++++++++----------- .../IO/WriteBufferFromAzureBlobStorage.cpp | 20 ++++----- .../isRetryableAzureException.cpp | 19 +++++++++ .../isRetryableAzureException.h | 14 +++++++ src/Storages/MergeTree/MergeTreeData.cpp | 27 +++--------- src/Storages/MergeTree/checkDataPart.cpp | 38 ++++++++--------- src/Storages/StorageAzureBlob.cpp | 3 +- 7 files changed, 81 insertions(+), 81 deletions(-) create mode 100644 src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.cpp create mode 100644 src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.h diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 5947b742339..68425c5ca18 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -3,6 +3,7 @@ #if USE_AZURE_BLOB_STORAGE #include +#include #include #include #include @@ -101,18 +102,6 @@ bool ReadBufferFromAzureBlobStorage::nextImpl() size_t sleep_time_with_backoff_milliseconds = 100; - auto handle_exception = [&, this](const auto & e, size_t i) - { - LOG_DEBUG(log, "Exception caught during Azure Read for file {} at attempt {}/{}: {}", path, i + 1, max_single_read_retries, e.Message); - if (i + 1 == max_single_read_retries) - throw; - - sleepForMilliseconds(sleep_time_with_backoff_milliseconds); - sleep_time_with_backoff_milliseconds *= 2; - initialized = false; - initialize(); - }; - for (size_t i = 0; i < max_single_read_retries; ++i) { try @@ -124,7 +113,14 @@ bool ReadBufferFromAzureBlobStorage::nextImpl() } catch (const Azure::Core::RequestFailedException & e) { - handle_exception(e, i); + LOG_DEBUG(log, "Exception caught during Azure Read for file {} at attempt {}/{}: {}", path, i + 1, max_single_read_retries, e.Message); + if (i + 1 == max_single_read_retries || !isRetryableAzureException(e)) + throw; + + sleepForMilliseconds(sleep_time_with_backoff_milliseconds); + sleep_time_with_backoff_milliseconds *= 2; + initialized = false; + initialize(); } } @@ -213,16 +209,6 @@ void ReadBufferFromAzureBlobStorage::initialize() size_t sleep_time_with_backoff_milliseconds = 100; - auto handle_exception = [&, this](const auto & e, size_t i) - { - LOG_DEBUG(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message); - if (i + 1 == max_single_download_retries) - throw; - - sleepForMilliseconds(sleep_time_with_backoff_milliseconds); - sleep_time_with_backoff_milliseconds *= 2; - }; - for (size_t i = 0; i < max_single_download_retries; ++i) { try @@ -233,7 +219,12 @@ void ReadBufferFromAzureBlobStorage::initialize() } catch (const Azure::Core::RequestFailedException & e) { - handle_exception(e,i); + LOG_DEBUG(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message); + if (i + 1 == max_single_download_retries || !isRetryableAzureException(e)) + throw; + + sleepForMilliseconds(sleep_time_with_backoff_milliseconds); + sleep_time_with_backoff_milliseconds *= 2; } } @@ -283,7 +274,7 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran catch (const Azure::Core::RequestFailedException & e) { LOG_DEBUG(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message); - if (i + 1 == max_single_download_retries) + if (i + 1 == max_single_download_retries || !isRetryableAzureException(e)) throw; sleepForMilliseconds(sleep_time_with_backoff_milliseconds); diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index fe64415191c..921f99ffef3 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -3,6 +3,7 @@ #if USE_AZURE_BLOB_STORAGE #include +#include #include #include #include @@ -83,17 +84,6 @@ WriteBufferFromAzureBlobStorage::~WriteBufferFromAzureBlobStorage() void WriteBufferFromAzureBlobStorage::execWithRetry(std::function func, size_t num_tries, size_t cost) { - auto handle_exception = [&, this](const auto & e, size_t i) - { - if (cost) - write_settings.resource_link.accumulate(cost); // Accumulate resource for later use, because we have failed to consume it - - if (i == num_tries - 1) - throw; - - LOG_DEBUG(log, "Write at attempt {} for blob `{}` failed: {} {}", i + 1, blob_path, e.what(), e.Message); - }; - for (size_t i = 0; i < num_tries; ++i) { try @@ -104,7 +94,13 @@ void WriteBufferFromAzureBlobStorage::execWithRetry(std::function func, } catch (const Azure::Core::RequestFailedException & e) { - handle_exception(e, i); + if (cost) + write_settings.resource_link.accumulate(cost); // Accumulate resource for later use, because we have failed to consume it + + if (i == num_tries - 1 || !isRetryableAzureException(e)) + throw; + + LOG_DEBUG(log, "Write at attempt {} for blob `{}` failed: {} {}", i + 1, blob_path, e.what(), e.Message); } catch (...) { diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.cpp new file mode 100644 index 00000000000..e32815e9613 --- /dev/null +++ b/src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.cpp @@ -0,0 +1,19 @@ +#include +#if USE_AZURE_BLOB_STORAGE + +namespace DB +{ + +bool isRetryableAzureRequestException(const Azure::Core::RequestFailedException & e) +{ + /// Always retry transport errors. + if (dynamic_cast(&e)) + return true; + + /// Retry other 5xx errors just in case. + return e.StatusCode >= Azure::Core::Http::HttpStatusCode::InternalServerError; +} + +#endif + +} diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.h b/src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.h new file mode 100644 index 00000000000..dfd13e4c98a --- /dev/null +++ b/src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.h @@ -0,0 +1,14 @@ +#pragma once +#include "config.h" + +#if USE_AZURE_BLOB_STORAGE +#include + +namespace DB +{ + +bool isRetryableAzureException(const Azure::Core::RequestFailedException & e); + +} + +#endif diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e5ace0e5969..1f2ed96f11b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -8,7 +8,6 @@ #include #include #include -#include "Common/logger_useful.h" #include #include #include @@ -1312,7 +1311,8 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart( /// during loading, such as "not enough memory" or network error. if (isRetryableException(std::current_exception())) throw; - LOG_DEBUG(log, "Failed to load data part {}, unknown exception", part_name); + + LOG_DEBUG(log, "Failed to load data part {} with exception: {}", part_name, getExceptionMessage(std::current_exception(), false)); mark_broken(); return res; } @@ -1343,6 +1343,7 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart( /// during loading, such as "not enough memory" or network error. if (isRetryableException(std::current_exception())) throw; + mark_broken(); return res; } @@ -1461,25 +1462,9 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPartWithRetries( if (try_no + 1 == max_tries) throw; - String exception_message; - try - { - rethrow_exception(exception_ptr); - } - catch (const Exception & e) - { - exception_message = e.message(); - } - #if USE_AZURE_BLOB_STORAGE - catch (const Azure::Core::RequestFailedException & e) - { - exception_message = e.Message; - } - #endif - - - LOG_DEBUG(log, "Failed to load data part {} at try {} with retryable error: {}. Will retry in {} ms", - part_name, try_no, exception_message, initial_backoff_ms); + LOG_DEBUG(log, + "Failed to load data part {} at try {} with retryable error: {}. Will retry in {} ms", + part_name, try_no, getExceptionMessage(exception_ptr, false), initial_backoff_ms); std::this_thread::sleep_for(std::chrono::milliseconds(initial_backoff_ms)); initial_backoff_ms = std::min(initial_backoff_ms * 2, max_backoff_ms); diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index d64568e0c3e..208da561118 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -16,11 +15,9 @@ #include #include #include +#include #include -#if USE_AZURE_BLOB_STORAGE -#include -#endif namespace CurrentMetrics { @@ -66,33 +63,28 @@ bool isRetryableException(std::exception_ptr exception_ptr) #if USE_AWS_S3 catch (const S3Exception & s3_exception) { - if (s3_exception.isRetryableError()) - return true; + return s3_exception.isRetryableError(); } #endif #if USE_AZURE_BLOB_STORAGE - catch (const Azure::Core::RequestFailedException &) + catch (const Azure::Core::RequestFailedException & e) { - return true; + return isRetryableAzureException(e); } #endif catch (const ErrnoException & e) { - if (e.getErrno() == EMFILE) - return true; + return e.getErrno() == EMFILE; } - catch (const Coordination::Exception & e) + catch (const Coordination::Exception & e) { - if (Coordination::isHardwareError(e.code)) - return true; + return Coordination::isHardwareError(e.code); } catch (const Exception & e) { - if (isNotEnoughMemoryErrorCode(e.code())) - return true; - - if (e.code() == ErrorCodes::NETWORK_ERROR || e.code() == ErrorCodes::SOCKET_TIMEOUT) - return true; + return isNotEnoughMemoryErrorCode(e.code()) + || e.code() == ErrorCodes::NETWORK_ERROR + || e.code() == ErrorCodes::SOCKET_TIMEOUT; } catch (const Poco::Net::NetException &) { @@ -102,10 +94,12 @@ bool isRetryableException(std::exception_ptr exception_ptr) { return true; } - - /// In fact, there can be other similar situations. - /// But it is OK, because there is a safety guard against deleting too many parts. - return false; + catch (...) + { + /// In fact, there can be other similar situations. + /// But it is OK, because there is a safety guard against deleting too many parts. + return false; + } } diff --git a/src/Storages/StorageAzureBlob.cpp b/src/Storages/StorageAzureBlob.cpp index 306a5eac8e5..86e96f29580 100644 --- a/src/Storages/StorageAzureBlob.cpp +++ b/src/Storages/StorageAzureBlob.cpp @@ -432,7 +432,8 @@ AzureClientPtr StorageAzureBlob::createClient(StorageAzureBlob::Configuration co try { result = std::make_unique(blob_service_client->CreateBlobContainer(configuration.container).Value); - } catch (const Azure::Storage::StorageException & e) + } + catch (const Azure::Storage::StorageException & e) { if (e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict && e.ReasonPhrase == "The specified container already exists.") From 0f5a3eae7e67c4b82d593aaf1d5f67d70a64d01d Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 4 Apr 2024 20:00:22 +0000 Subject: [PATCH 1005/1165] Better --- src/Functions/getScalar.cpp | 2 +- src/Interpreters/Context.cpp | 12 ++++++------ src/Interpreters/Context.h | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Functions/getScalar.cpp b/src/Functions/getScalar.cpp index d72c84b8528..7196cbc0a36 100644 --- a/src/Functions/getScalar.cpp +++ b/src/Functions/getScalar.cpp @@ -83,7 +83,7 @@ public: static ColumnWithTypeAndName createScalar(ContextPtr context_) { - if (const auto * block = context_->tryGetSpecialScalar(Scalar::scalar_name)) + if (auto block = context_->tryGetSpecialScalar(Scalar::scalar_name)) return block->getByPosition(0); else if (context_->hasQueryContext()) { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 0f28e5d8771..9c14a0485a3 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -48,7 +48,7 @@ #include #include #include -#include +//#include #include #include #include @@ -1550,14 +1550,14 @@ ClassifierPtr Context::getWorkloadClassifier() const } -const Scalars & Context::getScalars() const +Scalars Context::getScalars() const { std::lock_guard lock(mutex); return scalars; } -const Block & Context::getScalar(const String & name) const +Block Context::getScalar(const String & name) const { std::lock_guard lock(mutex); @@ -1571,13 +1571,13 @@ const Block & Context::getScalar(const String & name) const return it->second; } -const Block * Context::tryGetSpecialScalar(const String & name) const +std::optional Context::tryGetSpecialScalar(const String & name) const { std::lock_guard lock(mutex); auto it = special_scalars.find(name); if (special_scalars.end() == it) - return nullptr; - return &it->second; + return std::nullopt; + return it->second; } Tables Context::getExternalTables() const diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 8601d09621f..a0225cb2f9a 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -680,12 +680,12 @@ public: std::shared_ptr findExternalTable(const String & table_name) const; std::shared_ptr removeExternalTable(const String & table_name); - const Scalars & getScalars() const; - const Block & getScalar(const String & name) const; + Scalars getScalars() const; + Block getScalar(const String & name) const; void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; - const Block * tryGetSpecialScalar(const String & name) const; + std::optional tryGetSpecialScalar(const String & name) const; void addSpecialScalar(const String & name, const Block & block); const QueryAccessInfo & getQueryAccessInfo() const { return *getQueryAccessInfoPtr(); } From 7329b9509a89ad027dd038677780ec44660a9e76 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 4 Apr 2024 20:01:05 +0000 Subject: [PATCH 1006/1165] Fix header --- src/Interpreters/Context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 9c14a0485a3..faee0602f56 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -48,7 +48,7 @@ #include #include #include -//#include +#include #include #include #include From d7b7e7e79031fd69db8f47d6cd51663293d12cc8 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 4 Apr 2024 22:09:21 +0200 Subject: [PATCH 1007/1165] Improve comment. --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index fd5f3853a6b..36f927310f3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -799,8 +799,8 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: throw Exception( ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); - /// When we attach existing parts it's okay to be in read-only mode - /// For example during RESTORE REPLICA. + /// Usually parts should not be attached in read-only mode. So we retry until the table is not read-only. + /// However there is one case when it's necessary to attach in read-only mode - during execution of the RESTORE REPLICA command. if (!allow_attach_while_readonly) { retries_ctl.setUserError( From 43d898874453f9a3a8cac4e264605dfed38c788f Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 4 Apr 2024 22:10:06 +0200 Subject: [PATCH 1008/1165] Update tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py Co-authored-by: pufit --- .../integration/test_backup_restore_on_cluster/test_slow_rmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py b/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py index 987f8669488..211b92f5ea3 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py +++ b/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py @@ -82,7 +82,7 @@ def test_replicated_database_async(): node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' mydb.tbl") backup_name = new_backup_name() - [id, status] = node1.query( + id, status = node1.query( f"BACKUP DATABASE mydb ON CLUSTER 'cluster' TO {backup_name} ASYNC" ).split("\t") From 43687e57329e155a603fc9c427f55e42215ad19f Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 4 Apr 2024 22:10:16 +0200 Subject: [PATCH 1009/1165] Update tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py Co-authored-by: pufit --- .../integration/test_backup_restore_on_cluster/test_slow_rmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py b/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py index 211b92f5ea3..15c344eadf8 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py +++ b/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py @@ -96,7 +96,7 @@ def test_replicated_database_async(): node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' SYNC") - [id, status] = node1.query( + id, status = node1.query( f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} ASYNC" ).split("\t") From 1551ab7bd6b1562483157bd2496b535b612326f7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Apr 2024 23:21:00 +0200 Subject: [PATCH 1010/1165] Fix error --- packages/clickhouse-library-bridge.yaml | 8 ++++---- packages/clickhouse-odbc-bridge.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/clickhouse-library-bridge.yaml b/packages/clickhouse-library-bridge.yaml index 95e7d4aaad0..d041e7a26db 100644 --- a/packages/clickhouse-library-bridge.yaml +++ b/packages/clickhouse-library-bridge.yaml @@ -26,10 +26,10 @@ contents: dst: /usr/bin/clickhouse-library-bridge # docs - src: ../AUTHORS - dst: /usr/share/doc/clickhouse-common-static/AUTHORS + dst: /usr/share/doc/clickhouse-library-bridge/AUTHORS - src: ../CHANGELOG.md - dst: /usr/share/doc/clickhouse-common-static/CHANGELOG.md + dst: /usr/share/doc/clickhouse-library-bridge/CHANGELOG.md - src: ../LICENSE - dst: /usr/share/doc/clickhouse-common-static/LICENSE + dst: /usr/share/doc/clickhouse-library-bridge/LICENSE - src: ../README.md - dst: /usr/share/doc/clickhouse-common-static/README.md + dst: /usr/share/doc/clickhouse-library-bridge/README.md diff --git a/packages/clickhouse-odbc-bridge.yaml b/packages/clickhouse-odbc-bridge.yaml index 2a7edf41549..98c459c8c26 100644 --- a/packages/clickhouse-odbc-bridge.yaml +++ b/packages/clickhouse-odbc-bridge.yaml @@ -26,10 +26,10 @@ contents: dst: /usr/bin/clickhouse-odbc-bridge # docs - src: ../AUTHORS - dst: /usr/share/doc/clickhouse-common-static/AUTHORS + dst: /usr/share/doc/clickhouse-odbc-bridge/AUTHORS - src: ../CHANGELOG.md - dst: /usr/share/doc/clickhouse-common-static/CHANGELOG.md + dst: /usr/share/doc/clickhouse-odbc-bridge/CHANGELOG.md - src: ../LICENSE - dst: /usr/share/doc/clickhouse-common-static/LICENSE + dst: /usr/share/doc/clickhouse-odbc-bridge/LICENSE - src: ../README.md - dst: /usr/share/doc/clickhouse-common-static/README.md + dst: /usr/share/doc/clickhouse-odbc-bridge/README.md From b1bd34f66e82173bfc48c7e1a612a967562fcbc6 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Thu, 4 Apr 2024 20:25:49 +0000 Subject: [PATCH 1011/1165] fix --- src/Processors/QueryPlan/PartsSplitter.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 2af1bcb0260..ec51875587e 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -125,14 +126,18 @@ int compareValues(const Values & lhs, const Values & rhs) class IndexAccess { public: - explicit IndexAccess(const RangesInDataParts & parts_) : parts(parts_) { } + explicit IndexAccess(const RangesInDataParts & parts_) : parts(parts_) + { + for (const auto & part : parts) + loaded_columns = std::min(loaded_columns, part.data_part->getIndex().size()); + } Values getValue(size_t part_idx, size_t mark) const { const auto & index = parts[part_idx].data_part->getIndex(); - size_t size = index.size(); - Values values(size); - for (size_t i = 0; i < size; ++i) + chassert(index.size() >= loaded_columns); + Values values(loaded_columns); + for (size_t i = 0; i < loaded_columns; ++i) { index[i]->get(mark, values[i]); if (values[i].isNull()) @@ -199,6 +204,7 @@ public: } private: const RangesInDataParts & parts; + size_t loaded_columns = std::numeric_limits::max(); }; class RangesInDataPartsBuilder From 6be747bf32a7f1fcd9fee8f86c72dd2b03e48c02 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Thu, 4 Apr 2024 22:28:29 +0000 Subject: [PATCH 1012/1165] add test --- .../__init__.py | 0 .../test.py | 47 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/integration/test_final_bug_with_pk_columns_loading/__init__.py create mode 100644 tests/integration/test_final_bug_with_pk_columns_loading/test.py diff --git a/tests/integration/test_final_bug_with_pk_columns_loading/__init__.py b/tests/integration/test_final_bug_with_pk_columns_loading/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_final_bug_with_pk_columns_loading/test.py b/tests/integration/test_final_bug_with_pk_columns_loading/test.py new file mode 100644 index 00000000000..e710b9942dc --- /dev/null +++ b/tests/integration/test_final_bug_with_pk_columns_loading/test.py @@ -0,0 +1,47 @@ +import pytest +import logging + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", stay_alive=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + cluster.shutdown() + + +def test_simple_query_after_restart(start_cluster): + node.query( + """ + create table t(a UInt32, b UInt32) engine=MergeTree order by (a, b) settings index_granularity=1; + + insert into t select 42, number from numbers_mt(100); + insert into t select number, number from numbers_mt(100); + """ + ) + + node.restart_clickhouse() + + assert ( + int( + node.query( + "select count() from t where not ignore(*)", + settings={ + "max_threads": 4, + "merge_tree_min_bytes_for_concurrent_read": 1, + "merge_tree_min_rows_for_concurrent_read": 1, + "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability": 1, + }, + ) + ) + == 200 + ) From dd852da33925af8ba52a89034da774d512add241 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 4 Apr 2024 15:57:42 +0200 Subject: [PATCH 1013/1165] add more debug logs --- .../Net/include/Poco/Net/HTTPClientSession.h | 5 -- base/poco/Net/src/HTTPClientSession.cpp | 26 +++++-- base/poco/Net/src/HTTPMessage.cpp | 2 +- src/Common/HTTPConnectionPool.cpp | 68 ++++++++++++++++--- src/Disks/ObjectStorages/S3/diskSettings.cpp | 2 + src/IO/S3/Client.h | 8 +-- src/IO/S3/PocoHTTPClient.cpp | 3 +- src/IO/S3/PocoHTTPClient.h | 1 + src/IO/S3/tests/gtest_aws_s3_client.cpp | 2 +- 9 files changed, 91 insertions(+), 26 deletions(-) diff --git a/base/poco/Net/include/Poco/Net/HTTPClientSession.h b/base/poco/Net/include/Poco/Net/HTTPClientSession.h index b418937c4d5..cbf4619834b 100644 --- a/base/poco/Net/include/Poco/Net/HTTPClientSession.h +++ b/base/poco/Net/include/Poco/Net/HTTPClientSession.h @@ -458,11 +458,6 @@ namespace Net return _lastRequest; } - inline void HTTPClientSession::setLastRequest(Poco::Timestamp time) - { - _lastRequest = time; - } - inline double HTTPClientSession::getKeepAliveReliability() const { return _defaultKeepAliveReliabilityLevel; diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp index 59800232ba9..afa1eff68a2 100644 --- a/base/poco/Net/src/HTTPClientSession.cpp +++ b/base/poco/Net/src/HTTPClientSession.cpp @@ -223,12 +223,24 @@ void HTTPClientSession::setKeepAliveTimeout(const Poco::Timespan& timeout) { if (connected()) { - throw Poco::IllegalStateException("cannot change keep alive timeout on initiated connection"); + throw Poco::IllegalStateException("cannot change keep alive timeout on initiated connection, " + "That value is managed privately after connection is established."); } _keepAliveTimeout = timeout; } +void HTTPClientSession::setLastRequest(Poco::Timestamp time) +{ + if (connected()) + { + throw Poco::IllegalStateException("cannot change last request on initiated connection, " + "That value is managed privately after connection is established."); + } + _lastRequest = time; +} + + std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request) { _pRequestStream = 0; @@ -246,8 +258,8 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request) { if (!connected()) reconnect(); - if (!keepAlive) - request.setKeepAlive(false); + if (!request.has(HTTPMessage::CONNECTION)) + request.setKeepAlive(keepAlive); if (keepAlive && !request.has(HTTPMessage::CONNECTION_KEEP_ALIVE) && _keepAliveTimeout.totalSeconds() > 0) request.setKeepAliveTimeout(_keepAliveTimeout.totalSeconds()); if (!request.has(HTTPRequest::HOST) && !_host.empty()) @@ -528,14 +540,16 @@ void HTTPClientSession::assign(Poco::Net::HTTPClientSession & session) if (buffered()) throw Poco::LogicException("assign to a session with not empty buffered data"); - setLastRequest(session.getLastRequest()); + poco_assert(!connected()); + setResolvedHost(session.getResolvedHost()); setProxyConfig(session.getProxyConfig()); setTimeout(session.getConnectionTimeout(), session.getSendTimeout(), session.getReceiveTimeout()); setKeepAlive(session.getKeepAlive()); - if (!connected()) - setKeepAliveTimeout(session.getKeepAliveTimeout()); + + setLastRequest(session.getLastRequest()); + setKeepAliveTimeout(session.getKeepAliveTimeout()); attachSocket(session.detachSocket()); diff --git a/base/poco/Net/src/HTTPMessage.cpp b/base/poco/Net/src/HTTPMessage.cpp index 2f974b8bf0b..af743dfa2eb 100644 --- a/base/poco/Net/src/HTTPMessage.cpp +++ b/base/poco/Net/src/HTTPMessage.cpp @@ -182,7 +182,7 @@ bool HTTPMessage::getKeepAlive() const void HTTPMessage::setKeepAliveTimeout(int timeout) { - add(HTTPMessage::CONNECTION_KEEP_ALIVE, std::format("timeout={}", timeout)); + add(HTTPMessage::CONNECTION_KEEP_ALIVE, std::format("timeout={}, max=1000", timeout)); } diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index 2d3a87dda6b..f64d6658a55 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -193,6 +193,18 @@ public: return total_connections_in_group >= limits.store_limit; } + size_t getStored() const + { + std::lock_guard lock(mutex); + return total_connections_in_group; + } + + size_t getStoreLimit() const + { + std::lock_guard lock(mutex); + return limits.store_limit; + } + void atConnectionCreate() { std::lock_guard lock(mutex); @@ -221,12 +233,6 @@ public: } } - void atPoolDestroy(size_t connections) - { - std::lock_guard lock(mutex); - total_connections_in_group -= connections; - } - HTTPConnectionGroupType getType() const { return type; } const IHTTPConnectionPoolForEndpoint::Metrics & getMetrics() const { return metrics; } @@ -345,11 +351,29 @@ private: Session::flushRequest(); } + String printAllHeaders(Poco::Net::HTTPMessage & message) const + { + String out; + out.reserve(300); + for (auto & [k, v] : message) + { + out.append(fmt::format("<{}: {}> ", k, v)); + } + return out; + } + std::ostream & sendRequest(Poco::Net::HTTPRequest & request) override { std::ostream & result = Session::sendRequest(request); result.exceptions(std::ios::badbit); + // that line is for temporary debug, will be removed + LOG_INFO(log, "Send request to {} with: usage count {}, keep-alive timeout={}, headers: {}", + getTarget(), + usage_cnt, + Session::getKeepAliveTimeout().totalSeconds(), + printAllHeaders(request)); + request_stream = &result; request_stream_completed = false; @@ -368,9 +392,12 @@ private: // that line is for temporary debug, will be removed if (response.has(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE)) - LOG_INFO(log, "received keep alive header: {}, original was {}", + LOG_INFO(log, "Received response from {} with: usage count {}, keep alive header: {}, original ka {}, headers: {}", + getTarget(), + usage_cnt, response.get(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE, Poco::Net::HTTPMessage::EMPTY), - originKA); + originKA, + printAllHeaders(response)); response_stream = &result; response_stream_completed = false; @@ -415,8 +442,19 @@ private: group->atConnectionDestroy(); if (!isExpired) + { if (auto lock = pool.lock()) lock->atConnectionDestroy(*this); + } + else + { + Poco::Timestamp now; + LOG_INFO(log, "Expired connection to {} with: usage count {}, keep alive timeout: {}, last usage ago: {}", + getTarget(), + usage_cnt, + Session::getKeepAliveTimeout().totalSeconds(), + Poco::Timespan(now - Session::getLastRequest()).totalSeconds()); + } CurrentMetrics::sub(metrics.active_count); } @@ -459,6 +497,7 @@ private: ConnectionGroup::Ptr group; IHTTPConnectionPoolForEndpoint::Metrics metrics; bool isExpired = false; + size_t usage_cnt = 1; LoggerPtr log = getLogger("PooledConnection"); @@ -527,6 +566,8 @@ public: stored_connections.pop(); setTimeouts(*it, timeouts); + it->usage_cnt += 1; + ProfileEvents::increment(getMetrics().reused, 1); CurrentMetrics::sub(getMetrics().stored_count, 1); @@ -647,12 +688,23 @@ private: if (!connection.connected() || connection.mustReconnect() || !connection.isCompleted() || connection.buffered() || group->isStoreLimitReached()) { + Poco::Timestamp now; + LOG_INFO(getLogger("PooledConnection"), + "Reset connection to {} with: usage count {}, keep alive timeout: {}, last usage ago: {}, is completed {}, store limit reached {} as {}/{}", + getTarget(), + connection.usage_cnt, + connection.getKeepAliveTimeout().totalSeconds(), + Poco::Timespan(now - connection.getLastRequest()).totalSeconds(), + connection.isCompleted(), + group->isStoreLimitReached(), group->getStored(), group->getStoreLimit()); + ProfileEvents::increment(getMetrics().reset, 1); return; } auto connection_to_store = allocateNewConnection(); connection_to_store->assign(connection); + connection_to_store->usage_cnt = connection.usage_cnt; { MemoryTrackerSwitcher switcher{&total_memory_tracker}; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index df1ccbb32d9..7ce94699053 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -76,6 +76,8 @@ std::unique_ptr getClient( client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", S3::DEFAULT_CONNECT_TIMEOUT_MS); client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", S3::DEFAULT_REQUEST_TIMEOUT_MS); client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", S3::DEFAULT_MAX_CONNECTIONS); + client_configuration.http_keep_alive_timeout = config.getUInt(config_prefix + ".http_keep_alive_timeout", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT); + client_configuration.endpointOverride = uri.endpoint; client_configuration.s3_use_adaptive_timeouts = config.getBool( config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts); diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index c7bc727bf32..c79ec05c8c6 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -96,9 +96,9 @@ bool isS3ExpressEndpoint(const std::string & endpoint); struct ClientSettings { - bool use_virtual_addressing; + bool use_virtual_addressing = false; /// Disable checksum to avoid extra read of the input stream - bool disable_checksum; + bool disable_checksum = false; /// Should client send ComposeObject request after upload to GCS. /// /// Previously ComposeObject request was required to make Copy possible, @@ -108,8 +108,8 @@ struct ClientSettings /// /// Ability to enable it preserved since likely it is required for old /// files. - bool gcs_issue_compose_request; - bool is_s3express_bucket; + bool gcs_issue_compose_request = false; + bool is_s3express_bucket = false; }; /// Client that improves the client from the AWS SDK diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index a29a4b0b8ee..150b8146147 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -146,7 +146,8 @@ ConnectionTimeouts getTimeoutsFromConfiguration(const PocoHTTPClientConfiguratio .withSendTimeout(Poco::Timespan(client_configuration.requestTimeoutMs * 1000)) .withReceiveTimeout(Poco::Timespan(client_configuration.requestTimeoutMs * 1000)) .withTCPKeepAliveTimeout(Poco::Timespan( - client_configuration.enableTcpKeepAlive ? client_configuration.tcpKeepAliveIntervalMs * 1000 : 0)); + client_configuration.enableTcpKeepAlive ? client_configuration.tcpKeepAliveIntervalMs * 1000 : 0)) + .withHTTPKeepAliveTimeout(Poco::Timespan(client_configuration.http_keep_alive_timeout, 0)); } PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_configuration) diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index ebbddbb2c7e..f568eb5ddb8 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -51,6 +51,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration /// See PoolBase::BehaviourOnLimit bool s3_use_adaptive_timeouts = true; + size_t http_keep_alive_timeout = DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT; std::function error_report; diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index 25786619241..0a28c578f69 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -159,7 +159,7 @@ void testServerSideEncryption( DB::S3::CredentialsConfiguration { .use_environment_credentials = use_environment_credentials, - .use_insecure_imds_request = use_insecure_imds_request + .use_insecure_imds_request = use_insecure_imds_request, } ); From 5cab8d185fb5ad1f8607a4ad7140a15469754e99 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 4 Apr 2024 19:29:42 +0200 Subject: [PATCH 1014/1165] more details --- base/poco/Net/src/HTTPClientSession.cpp | 2 +- src/Common/HTTPConnectionPool.cpp | 42 ++++++++++++++++--------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp index afa1eff68a2..bc70559c5eb 100644 --- a/base/poco/Net/src/HTTPClientSession.cpp +++ b/base/poco/Net/src/HTTPClientSession.cpp @@ -348,7 +348,7 @@ std::istream& HTTPClientSession::receiveResponse(HTTPResponse& response) /// when server sends its keep alive timeout, client has to follow that value auto timeout = response.getKeepAliveTimeout(); if (timeout > 0) - _keepAliveTimeout = Poco::Timespan(timeout, 0); + _keepAliveTimeout = std::min(_keepAliveTimeout, Poco::Timespan(timeout, 0)); } if (!_expectResponseBody || response.getStatus() < 200 || response.getStatus() == HTTPResponse::HTTP_NO_CONTENT || response.getStatus() == HTTPResponse::HTTP_NOT_MODIFIED) diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index f64d6658a55..eb6ce00e611 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -322,6 +322,11 @@ private: Session::getPort()); } + Poco::Timespan idleTime() { + Poco::Timestamp now; + return now - Session::getLastRequest(); + } + void flushRequest() override { if (bool(request_stream)) @@ -364,14 +369,18 @@ private: std::ostream & sendRequest(Poco::Net::HTTPRequest & request) override { + auto idle = idleTime(); std::ostream & result = Session::sendRequest(request); result.exceptions(std::ios::badbit); // that line is for temporary debug, will be removed - LOG_INFO(log, "Send request to {} with: usage count {}, keep-alive timeout={}, headers: {}", + LOG_INFO(log, "Send request to {} with: version {}, method {}, usage count {}, keep-alive timeout={}, last usage ago: {}ms, headers: {}", + request.getVersion(), + request.getMethod(), getTarget(), usage_cnt, Session::getKeepAliveTimeout().totalSeconds(), + idle.totalMilliseconds(), printAllHeaders(request)); request_stream = &result; @@ -391,13 +400,15 @@ private: result.exceptions(std::ios::badbit); // that line is for temporary debug, will be removed - if (response.has(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE)) - LOG_INFO(log, "Received response from {} with: usage count {}, keep alive header: {}, original ka {}, headers: {}", - getTarget(), - usage_cnt, - response.get(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE, Poco::Net::HTTPMessage::EMPTY), - originKA, - printAllHeaders(response)); + LOG_INFO(log, "Received response from {} with: version {}, code {}, usage count {}, keep alive header: {}, original ka {}, last usage ago: {}ms, headers: {}", + getTarget(), + response.getVersion(), + int(response.getStatus()), + usage_cnt, + response.get(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE, Poco::Net::HTTPMessage::EMPTY), + originKA, + idleTime().totalMilliseconds(), + printAllHeaders(response)); response_stream = &result; response_stream_completed = false; @@ -449,11 +460,11 @@ private: else { Poco::Timestamp now; - LOG_INFO(log, "Expired connection to {} with: usage count {}, keep alive timeout: {}, last usage ago: {}", + LOG_INFO(log, "Expired connection to {} with: usage count {}, keep alive timeout: {}, last usage ago: {}s", getTarget(), usage_cnt, Session::getKeepAliveTimeout().totalSeconds(), - Poco::Timespan(now - Session::getLastRequest()).totalSeconds()); + idleTime().totalSeconds()); } CurrentMetrics::sub(metrics.active_count); @@ -498,6 +509,7 @@ private: IHTTPConnectionPoolForEndpoint::Metrics metrics; bool isExpired = false; size_t usage_cnt = 1; + size_t exception_level = std::uncaught_exceptions(); LoggerPtr log = getLogger("PooledConnection"); @@ -568,7 +580,6 @@ public: setTimeouts(*it, timeouts); it->usage_cnt += 1; - ProfileEvents::increment(getMetrics().reused, 1); CurrentMetrics::sub(getMetrics().stored_count, 1); @@ -690,13 +701,16 @@ private: { Poco::Timestamp now; LOG_INFO(getLogger("PooledConnection"), - "Reset connection to {} with: usage count {}, keep alive timeout: {}, last usage ago: {}, is completed {}, store limit reached {} as {}/{}", + "Reset connection to {} with: usage count {}, keep alive timeout: {}, connected {}, must recon {}, last usage ago: {}, is completed {}, store limit reached {} as {}/{}, there is exception {}", getTarget(), connection.usage_cnt, connection.getKeepAliveTimeout().totalSeconds(), - Poco::Timespan(now - connection.getLastRequest()).totalSeconds(), + connection.connected(), + connection.mustReconnect(), + connection.idleTime().totalSeconds(), connection.isCompleted(), - group->isStoreLimitReached(), group->getStored(), group->getStoreLimit()); + group->isStoreLimitReached(), group->getStored(), group->getStoreLimit(), + connection.exception_level - std::uncaught_exceptions()); ProfileEvents::increment(getMetrics().reset, 1); return; From ae3a1999398b4f16880e2d892cb11bb414944b81 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 4 Apr 2024 22:49:52 +0200 Subject: [PATCH 1015/1165] support max requests for keep alive --- .../Net/include/Poco/Net/HTTPClientSession.h | 20 ++++ base/poco/Net/include/Poco/Net/HTTPMessage.h | 3 +- base/poco/Net/src/HTTPClientSession.cpp | 33 +++++- base/poco/Net/src/HTTPMessage.cpp | 37 ++++--- src/Common/HTTPConnectionPool.cpp | 69 +++++++----- src/Common/tests/gtest_connection_pool.cpp | 103 ++++++++++++++++-- src/Core/Defines.h | 1 + src/Disks/ObjectStorages/S3/diskSettings.cpp | 3 +- src/IO/ConnectionTimeouts.cpp | 1 + src/IO/ConnectionTimeouts.h | 2 + src/IO/S3/Credentials.h | 2 + src/IO/S3/PocoHTTPClient.h | 1 + 12 files changed, 219 insertions(+), 56 deletions(-) diff --git a/base/poco/Net/include/Poco/Net/HTTPClientSession.h b/base/poco/Net/include/Poco/Net/HTTPClientSession.h index cbf4619834b..edbb135d8c6 100644 --- a/base/poco/Net/include/Poco/Net/HTTPClientSession.h +++ b/base/poco/Net/include/Poco/Net/HTTPClientSession.h @@ -213,6 +213,12 @@ namespace Net Poco::Timespan getKeepAliveTimeout() const; /// Returns the connection timeout for HTTP connections. + void setKeepAliveMaxRequests(int max_requests); + + int getKeepAliveMaxRequests() const; + + int getKeepAliveRequest() const; + bool isKeepAliveExpired(double reliability = 1.0) const; /// Returns if the connection is expired with some margin as fraction of timeout as reliability @@ -352,6 +358,8 @@ namespace Net void assign(HTTPClientSession & session); + void setKeepAliveRequest(int request); + HTTPSessionFactory _proxySessionFactory; /// Factory to create HTTPClientSession to proxy. private: @@ -360,6 +368,8 @@ namespace Net Poco::UInt16 _port; ProxyConfig _proxyConfig; Poco::Timespan _keepAliveTimeout; + int _keepAliveCurrentRequest = 0; + int _keepAliveMaxRequests = 1000; Poco::Timestamp _lastRequest; bool _reconnect; bool _mustReconnect; @@ -463,6 +473,16 @@ namespace Net return _defaultKeepAliveReliabilityLevel; } + inline int HTTPClientSession::getKeepAliveMaxRequests() const + { + return _keepAliveMaxRequests; + } + + inline int HTTPClientSession::getKeepAliveRequest() const + { + return _keepAliveCurrentRequest; + } + } } // namespace Poco::Net diff --git a/base/poco/Net/include/Poco/Net/HTTPMessage.h b/base/poco/Net/include/Poco/Net/HTTPMessage.h index 994807ffbff..8bc95ccc1af 100644 --- a/base/poco/Net/include/Poco/Net/HTTPMessage.h +++ b/base/poco/Net/include/Poco/Net/HTTPMessage.h @@ -120,8 +120,9 @@ namespace Net /// The value is set to "Keep-Alive" if keepAlive is /// true, or to "Close" otherwise. - void setKeepAliveTimeout(int timeout); + void setKeepAliveTimeout(int timeout, int max_requests); int getKeepAliveTimeout() const; + int getKeepAliveMaxRequests() const; bool getKeepAlive() const; /// Returns true if diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp index bc70559c5eb..e489ab56b98 100644 --- a/base/poco/Net/src/HTTPClientSession.cpp +++ b/base/poco/Net/src/HTTPClientSession.cpp @@ -230,7 +230,25 @@ void HTTPClientSession::setKeepAliveTimeout(const Poco::Timespan& timeout) } -void HTTPClientSession::setLastRequest(Poco::Timestamp time) +void HTTPClientSession::setKeepAliveMaxRequests(int max_requests) +{ + if (connected()) + { + throw Poco::IllegalStateException("cannot change keep alive max requests on initiated connection, " + "That value is managed privately after connection is established."); + } + _keepAliveMaxRequests = max_requests; +} + + +void HTTPClientSession::setKeepAliveRequest(int request) +{ + _keepAliveCurrentRequest = request; +} + + + + void HTTPClientSession::setLastRequest(Poco::Timestamp time) { if (connected()) { @@ -248,6 +266,8 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request) clearException(); _responseReceived = false; + _keepAliveCurrentRequest += 1; + bool keepAlive = getKeepAlive(); if (((connected() && !keepAlive) || mustReconnect()) && !_host.empty()) { @@ -261,7 +281,7 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request) if (!request.has(HTTPMessage::CONNECTION)) request.setKeepAlive(keepAlive); if (keepAlive && !request.has(HTTPMessage::CONNECTION_KEEP_ALIVE) && _keepAliveTimeout.totalSeconds() > 0) - request.setKeepAliveTimeout(_keepAliveTimeout.totalSeconds()); + request.setKeepAliveTimeout(_keepAliveTimeout.totalSeconds(), _keepAliveMaxRequests); if (!request.has(HTTPRequest::HOST) && !_host.empty()) request.setHost(_host, _port); if (!_proxyConfig.host.empty() && !bypassProxy()) @@ -349,6 +369,9 @@ std::istream& HTTPClientSession::receiveResponse(HTTPResponse& response) auto timeout = response.getKeepAliveTimeout(); if (timeout > 0) _keepAliveTimeout = std::min(_keepAliveTimeout, Poco::Timespan(timeout, 0)); + auto max_requests = response.getKeepAliveMaxRequests(); + if (max_requests > 0) + _keepAliveMaxRequests = std::min(_keepAliveMaxRequests, max_requests); } if (!_expectResponseBody || response.getStatus() < 200 || response.getStatus() == HTTPResponse::HTTP_NO_CONTENT || response.getStatus() == HTTPResponse::HTTP_NOT_MODIFIED) @@ -460,7 +483,8 @@ std::string HTTPClientSession::proxyRequestPrefix() const bool HTTPClientSession::isKeepAliveExpired(double reliability) const { Poco::Timestamp now; - return Timespan(Timestamp::TimeDiff(reliability *_keepAliveTimeout.totalMicroseconds())) <= now - _lastRequest; + return Timespan(Timestamp::TimeDiff(reliability *_keepAliveTimeout.totalMicroseconds())) <= now - _lastRequest + || _keepAliveCurrentRequest > _keepAliveMaxRequests; } bool HTTPClientSession::mustReconnect() const @@ -551,6 +575,9 @@ void HTTPClientSession::assign(Poco::Net::HTTPClientSession & session) setLastRequest(session.getLastRequest()); setKeepAliveTimeout(session.getKeepAliveTimeout()); + _keepAliveMaxRequests = session._keepAliveMaxRequests; + _keepAliveCurrentRequest = session._keepAliveCurrentRequest; + attachSocket(session.detachSocket()); session.reset(); diff --git a/base/poco/Net/src/HTTPMessage.cpp b/base/poco/Net/src/HTTPMessage.cpp index af743dfa2eb..c0083ec410c 100644 --- a/base/poco/Net/src/HTTPMessage.cpp +++ b/base/poco/Net/src/HTTPMessage.cpp @@ -180,27 +180,25 @@ bool HTTPMessage::getKeepAlive() const } -void HTTPMessage::setKeepAliveTimeout(int timeout) +void HTTPMessage::setKeepAliveTimeout(int timeout, int max_requests) { - add(HTTPMessage::CONNECTION_KEEP_ALIVE, std::format("timeout={}, max=1000", timeout)); + add(HTTPMessage::CONNECTION_KEEP_ALIVE, std::format("timeout={}, max={}", timeout, max_requests)); } -int parseTimeoutFromHeaderValue(const std::string_view header_value) +int parseFromHeaderValues(const std::string_view header_value, const std::string_view param_name) { - static const std::string_view timeout_param = "timeout="; + auto param_value_pos = header_value.find(param_name); + if (param_value_pos == std::string::npos) + param_value_pos = header_value.size(); + if (param_value_pos != header_value.size()) + param_value_pos += param_name.size(); - auto timeout_pos = header_value.find(timeout_param); - if (timeout_pos == std::string::npos) - timeout_pos = header_value.size(); - if (timeout_pos != header_value.size()) - timeout_pos += timeout_param.size(); + auto param_value_end = header_value.find(',', param_value_pos); + if (param_value_end == std::string::npos) + param_value_end = header_value.size(); - auto timeout_end = header_value.find(',', timeout_pos); - if (timeout_end == std::string::npos) - timeout_end = header_value.size(); - - auto timeout_value_substr = header_value.substr(timeout_pos, timeout_end - timeout_pos); + auto timeout_value_substr = header_value.substr(param_value_pos, param_value_end - param_value_pos); if (timeout_value_substr.empty()) return -1; @@ -217,7 +215,16 @@ int parseTimeoutFromHeaderValue(const std::string_view header_value) int HTTPMessage::getKeepAliveTimeout() const { const std::string& ka_header = get(HTTPMessage::CONNECTION_KEEP_ALIVE, HTTPMessage::EMPTY); - return parseTimeoutFromHeaderValue(ka_header); + static const std::string_view timeout_param = "timeout="; + return parseFromHeaderValues(ka_header, timeout_param); +} + + +int HTTPMessage::getKeepAliveMaxRequests() const +{ + const std::string& ka_header = get(HTTPMessage::CONNECTION_KEEP_ALIVE, HTTPMessage::EMPTY); + static const std::string_view timeout_param = "max="; + return parseFromHeaderValues(ka_header, timeout_param); } } } // namespace Poco::Net diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index eb6ce00e611..926222934e4 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -301,6 +301,8 @@ private: auto timeouts = getTimeouts(*this); auto new_connection = lock->getConnection(timeouts); Session::assign(*new_connection); + if (Session::getKeepAliveRequest() == 0) + Session::setKeepAliveRequest(1); } else { @@ -322,7 +324,8 @@ private: Session::getPort()); } - Poco::Timespan idleTime() { + Poco::Timespan idleTime() + { Poco::Timestamp now; return now - Session::getLastRequest(); } @@ -374,11 +377,11 @@ private: result.exceptions(std::ios::badbit); // that line is for temporary debug, will be removed - LOG_INFO(log, "Send request to {} with: version {}, method {}, usage count {}, keep-alive timeout={}, last usage ago: {}ms, headers: {}", + LOG_INFO(log, "Send request to {} with: version {}, method {}, request no {}, keep-alive timeout={}, last usage ago: {}ms, headers: {}", request.getVersion(), request.getMethod(), getTarget(), - usage_cnt, + Session::getKeepAliveRequest(), Session::getKeepAliveTimeout().totalSeconds(), idle.totalMilliseconds(), printAllHeaders(request)); @@ -400,11 +403,11 @@ private: result.exceptions(std::ios::badbit); // that line is for temporary debug, will be removed - LOG_INFO(log, "Received response from {} with: version {}, code {}, usage count {}, keep alive header: {}, original ka {}, last usage ago: {}ms, headers: {}", + LOG_INFO(log, "Received response from {} with: version {}, code {}, request no {}, keep alive header: {}, original ka {}, last usage ago: {}ms, headers: {}", getTarget(), response.getVersion(), int(response.getStatus()), - usage_cnt, + Session::getKeepAliveRequest(), response.get(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE, Poco::Net::HTTPMessage::EMPTY), originKA, idleTime().totalMilliseconds(), @@ -460,9 +463,9 @@ private: else { Poco::Timestamp now; - LOG_INFO(log, "Expired connection to {} with: usage count {}, keep alive timeout: {}, last usage ago: {}s", + LOG_INFO(log, "Expired connection to {} with: request no {}, keep alive timeout: {}, last usage ago: {}s", getTarget(), - usage_cnt, + Session::getKeepAliveRequest(), Session::getKeepAliveTimeout().totalSeconds(), idleTime().totalSeconds()); } @@ -474,8 +477,15 @@ private: friend class EndpointConnectionPool; template - explicit PooledConnection(EndpointConnectionPool::WeakPtr pool_, ConnectionGroup::Ptr group_, IHTTPConnectionPoolForEndpoint::Metrics metrics_, Args &&... args) - : Session(args...), pool(std::move(pool_)), group(group_), metrics(std::move(metrics_)) + explicit PooledConnection( + EndpointConnectionPool::WeakPtr pool_, + ConnectionGroup::Ptr group_, + IHTTPConnectionPoolForEndpoint::Metrics metrics_, + Args &&... args) + : Session(args...) + , pool(std::move(pool_)) + , group(group_) + , metrics(std::move(metrics_)) { CurrentMetrics::add(metrics.active_count); group->atConnectionCreate(); @@ -508,7 +518,7 @@ private: ConnectionGroup::Ptr group; IHTTPConnectionPoolForEndpoint::Metrics metrics; bool isExpired = false; - size_t usage_cnt = 1; + size_t exception_level = std::uncaught_exceptions(); LoggerPtr log = getLogger("PooledConnection"); @@ -578,7 +588,6 @@ public: stored_connections.pop(); setTimeouts(*it, timeouts); - it->usage_cnt += 1; ProfileEvents::increment(getMetrics().reused, 1); CurrentMetrics::sub(getMetrics().stored_count, 1); @@ -655,47 +664,50 @@ private: return connection->isKeepAliveExpired(0.8); } - ConnectionPtr allocateNewConnection() + + ConnectionPtr prepareNewConnection(const ConnectionTimeouts & timeouts) { - ConnectionPtr connection = PooledConnection::create(this->getWeakFromThis(), group, getMetrics(), host, port); + auto connection = PooledConnection::create(this->getWeakFromThis(), group, getMetrics(), host, port); + connection->setKeepAlive(true); + setTimeouts(*connection, timeouts); if (!proxy_configuration.isEmpty()) { connection->setProxyConfig(proxyConfigurationToPocoProxyConfig(proxy_configuration)); } - return connection; - } - - ConnectionPtr prepareNewConnection(const ConnectionTimeouts & timeouts) - { auto address = HostResolversPool::instance().getResolver(host)->resolve(); - - auto session = allocateNewConnection(); - - setTimeouts(*session, timeouts); - session->setResolvedHost(*address); + connection->setResolvedHost(*address); try { auto timer = CurrentThread::getProfileEvents().timer(getMetrics().elapsed_microseconds); - session->doConnect(); + connection->doConnect(); } catch (...) { address.setFail(); ProfileEvents::increment(getMetrics().errors); - session->reset(); + connection->reset(); throw; } ProfileEvents::increment(getMetrics().created); - return session; + return connection; } void atConnectionDestroy(PooledConnection & connection) { + if (connection.getKeepAliveRequest() >= connection.getKeepAliveMaxRequests()) + { + LOG_INFO(getLogger("PooledConnection"), "Expired by connection number {}", + connection.getKeepAliveRequest()); + + ProfileEvents::increment(getMetrics().expired, 1); + return; + } + if (!connection.connected() || connection.mustReconnect() || !connection.isCompleted() || connection.buffered() || group->isStoreLimitReached()) { @@ -703,7 +715,7 @@ private: LOG_INFO(getLogger("PooledConnection"), "Reset connection to {} with: usage count {}, keep alive timeout: {}, connected {}, must recon {}, last usage ago: {}, is completed {}, store limit reached {} as {}/{}, there is exception {}", getTarget(), - connection.usage_cnt, + connection.getKeepAliveRequest(), connection.getKeepAliveTimeout().totalSeconds(), connection.connected(), connection.mustReconnect(), @@ -716,9 +728,8 @@ private: return; } - auto connection_to_store = allocateNewConnection(); + auto connection_to_store = PooledConnection::create(this->getWeakFromThis(), group, getMetrics(), host, port); connection_to_store->assign(connection); - connection_to_store->usage_cnt = connection.usage_cnt; { MemoryTrackerSwitcher switcher{&total_memory_tracker}; diff --git a/src/Common/tests/gtest_connection_pool.cpp b/src/Common/tests/gtest_connection_pool.cpp index 36bf8bc7dae..cc091d12bb0 100644 --- a/src/Common/tests/gtest_connection_pool.cpp +++ b/src/Common/tests/gtest_connection_pool.cpp @@ -47,6 +47,7 @@ struct RequestOptions { size_t slowdown_receive = 0; int overwrite_keep_alive_timeout = 0; + int overwrite_keep_alive_max_requests = 10; }; size_t stream_copy_n(std::istream & in, std::ostream & out, std::size_t count = std::numeric_limits::max()) @@ -89,8 +90,10 @@ public: int value = request.getKeepAliveTimeout(); ASSERT_GT(value, 0); - if (options->get().overwrite_keep_alive_timeout > 0) - response.setKeepAliveTimeout(options->get().overwrite_keep_alive_timeout); + auto params = options->get(); + + if (params.overwrite_keep_alive_timeout > 0) + response.setKeepAliveTimeout(params.overwrite_keep_alive_timeout, params.overwrite_keep_alive_max_requests); response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); auto size = request.getContentLength(); @@ -99,8 +102,8 @@ public: else response.setChunkedTransferEncoding(true); // or chunk encoding - if (options->get().slowdown_receive > 0) - sleepForSeconds(options->get().slowdown_receive); + if (params.slowdown_receive > 0) + sleepForSeconds(params.slowdown_receive); stream_copy_n(request.stream(), response.send(), size); } @@ -189,10 +192,11 @@ protected: options->set(std::move(opt)); } - void setOverWriteTimeout(size_t seconds) + void setOverWriteKeepAlive(size_t seconds, int max_requests) { auto opt = options->get(); opt.overwrite_keep_alive_timeout = int(seconds); + opt.overwrite_keep_alive_max_requests= max_requests; options->set(std::move(opt)); } @@ -794,7 +798,7 @@ TEST_F(ConnectionPoolTest, ServerOverwriteKeepAlive) } { - setOverWriteTimeout(1); + setOverWriteKeepAlive(1, 10); auto connection = pool->getConnection(timeouts); echoRequest("Hello", *connection); ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds()); @@ -803,7 +807,7 @@ TEST_F(ConnectionPoolTest, ServerOverwriteKeepAlive) { // server do not overwrite it in the following requests but client has to remember last agreed value - setOverWriteTimeout(0); + setOverWriteKeepAlive(0, 0); auto connection = pool->getConnection(timeouts); echoRequest("Hello", *connection); ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds()); @@ -819,3 +823,88 @@ TEST_F(ConnectionPoolTest, ServerOverwriteKeepAlive) ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count)); } + +TEST_F(ConnectionPoolTest, MaxRequests) +{ + auto ka = Poco::Timespan(30, 0); // 30 seconds + timeouts.withHTTPKeepAliveTimeout(ka); + auto max_requests = 5; + timeouts.http_keep_alive_max_requests = max_requests; + + auto pool = getPool(); + auto metrics = pool->getMetrics(); + + for (int i = 1; i <= max_requests - 1; ++i) + { + auto connection = pool->getConnection(timeouts); + echoRequest("Hello", *connection); + ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds()); + ASSERT_EQ(max_requests, connection->getKeepAliveMaxRequests()); + ASSERT_EQ(i, connection->getKeepAliveRequest()); + } + + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(max_requests-1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(max_requests-2, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count)); + + { + auto connection = pool->getConnection(timeouts); + echoRequest("Hello", *connection); + ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds()); + ASSERT_EQ(max_requests, connection->getKeepAliveMaxRequests()); + ASSERT_EQ(max_requests, connection->getKeepAliveRequest()); + } + + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(max_requests-1, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(max_requests-1, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); +} + + +TEST_F(ConnectionPoolTest, ServerOverwriteMaxRequests) +{ + auto ka = Poco::Timespan(30, 0); // 30 seconds + timeouts.withHTTPKeepAliveTimeout(ka); + + auto pool = getPool(); + auto metrics = pool->getMetrics(); + + { + auto connection = pool->getConnection(timeouts); + echoRequest("Hello", *connection); + ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds()); + ASSERT_EQ(1000, connection->getKeepAliveMaxRequests()); + ASSERT_EQ(1, connection->getKeepAliveRequest()); + } + + auto max_requests = 3; + setOverWriteKeepAlive(5, max_requests); + + for (int i = 2; i <= 10*max_requests; ++i) + { + auto connection = pool->getConnection(timeouts); + echoRequest("Hello", *connection); + ASSERT_EQ(5, connection->getKeepAliveTimeout().totalSeconds()); + ASSERT_EQ(max_requests, connection->getKeepAliveMaxRequests()); + ASSERT_EQ(((i-1) % max_requests) + 1, connection->getKeepAliveRequest()); + } + + ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.created]); + ASSERT_EQ(10*max_requests-10, DB::CurrentThread::getProfileEvents()[metrics.preserved]); + ASSERT_EQ(10*max_requests-10, DB::CurrentThread::getProfileEvents()[metrics.reused]); + ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]); + ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.expired]); + + ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count)); + ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count)); +} diff --git a/src/Core/Defines.h b/src/Core/Defines.h index a8dd26519c2..f2142bc764d 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -54,6 +54,7 @@ static constexpr auto DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT = 15; static constexpr auto DEFAULT_TCP_KEEP_ALIVE_TIMEOUT = 290; static constexpr auto DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT = 30; +static constexpr auto DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST = 1000; static constexpr auto DBMS_DEFAULT_PATH = "/var/lib/clickhouse/"; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 7ce94699053..c3114eb0b6f 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -76,7 +76,8 @@ std::unique_ptr getClient( client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", S3::DEFAULT_CONNECT_TIMEOUT_MS); client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", S3::DEFAULT_REQUEST_TIMEOUT_MS); client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", S3::DEFAULT_MAX_CONNECTIONS); - client_configuration.http_keep_alive_timeout = config.getUInt(config_prefix + ".http_keep_alive_timeout", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT); + client_configuration.http_keep_alive_timeout = config.getUInt(config_prefix + ".http_keep_alive_timeout", S3::DEFAULT_KEEP_ALIVE_TIMEOUT); + client_configuration.http_keep_alive_max_requests = config.getUInt(config_prefix + ".http_keep_alive_max_requests", S3::DEFAULT_KEEP_ALIVE_MAX_REQUESTS); client_configuration.endpointOverride = uri.endpoint; client_configuration.s3_use_adaptive_timeouts = config.getBool( diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index 8813c958185..da6214ae477 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -148,6 +148,7 @@ void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeout if (!session.connected()) { session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); + session.setKeepAliveMaxRequests(int(timeouts.http_keep_alive_max_requests)); } } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index 49305f42d85..f497285bd0c 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -35,6 +35,8 @@ struct ConnectionTimeouts Poco::Timespan tcp_keep_alive_timeout = Poco::Timespan(DEFAULT_TCP_KEEP_ALIVE_TIMEOUT, 0); Poco::Timespan http_keep_alive_timeout = Poco::Timespan(DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT, 0); + size_t http_keep_alive_max_requests = DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST; + /// Timeouts for HedgedConnections Poco::Timespan hedged_connection_timeout = Poco::Timespan(DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, 0); diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index 34dc0c1d2bd..8d586223035 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -22,6 +22,8 @@ inline static constexpr uint64_t DEFAULT_EXPIRATION_WINDOW_SECONDS = 120; inline static constexpr uint64_t DEFAULT_CONNECT_TIMEOUT_MS = 1000; inline static constexpr uint64_t DEFAULT_REQUEST_TIMEOUT_MS = 30000; inline static constexpr uint64_t DEFAULT_MAX_CONNECTIONS = 100; +inline static constexpr uint64_t DEFAULT_KEEP_ALIVE_TIMEOUT = 5; +inline static constexpr uint64_t DEFAULT_KEEP_ALIVE_MAX_REQUESTS = 100; /// In GCP metadata service can be accessed via DNS regardless of IPv4 or IPv6. static inline constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata.google.internal"; diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index f568eb5ddb8..a0b35e9b4a9 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -52,6 +52,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration /// See PoolBase::BehaviourOnLimit bool s3_use_adaptive_timeouts = true; size_t http_keep_alive_timeout = DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT; + size_t http_keep_alive_max_requests = DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST; std::function error_report; From dddb0d9f4a83569e9a64952b20acfc95da2cdf24 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 5 Apr 2024 03:02:45 +0200 Subject: [PATCH 1016/1165] fix http_keep_alive_max_requests set up --- src/IO/ConnectionTimeouts.h | 7 +++++++ src/IO/S3/PocoHTTPClient.cpp | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index f497285bd0c..b86ec44d21c 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -71,6 +71,7 @@ APPLY_FOR_ALL_CONNECTION_TIMEOUT_MEMBERS(DECLARE_BUILDER_FOR_MEMBER) ConnectionTimeouts & withConnectionTimeout(size_t seconds); ConnectionTimeouts & withConnectionTimeout(Poco::Timespan span); + ConnectionTimeouts & withHTTPKeepAliveMaxRequests(size_t requests); }; /// NOLINTBEGIN(bugprone-macro-parentheses) @@ -116,6 +117,12 @@ inline ConnectionTimeouts & ConnectionTimeouts::withConnectionTimeout(Poco::Time return *this; } +inline ConnectionTimeouts & ConnectionTimeouts::withHTTPKeepAliveMaxRequests(size_t requests) +{ + http_keep_alive_max_requests = requests; + return *this; +} + void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts); ConnectionTimeouts getTimeouts(const Poco::Net::HTTPClientSession & session); diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 150b8146147..de20a712d4c 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -147,7 +147,8 @@ ConnectionTimeouts getTimeoutsFromConfiguration(const PocoHTTPClientConfiguratio .withReceiveTimeout(Poco::Timespan(client_configuration.requestTimeoutMs * 1000)) .withTCPKeepAliveTimeout(Poco::Timespan( client_configuration.enableTcpKeepAlive ? client_configuration.tcpKeepAliveIntervalMs * 1000 : 0)) - .withHTTPKeepAliveTimeout(Poco::Timespan(client_configuration.http_keep_alive_timeout, 0)); + .withHTTPKeepAliveTimeout(Poco::Timespan(client_configuration.http_keep_alive_timeout, 0)) + .withHTTPKeepAliveMaxRequests(client_configuration.http_keep_alive_max_requests); } PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_configuration) From cf982cc114ef5b226815360590e2c207516de658 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 5 Apr 2024 05:00:01 +0200 Subject: [PATCH 1017/1165] remove debug logging --- src/Common/HTTPConnectionPool.cpp | 76 ------------------------------- 1 file changed, 76 deletions(-) diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index 81c36dcd970..7f99d6a647f 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -193,18 +193,6 @@ public: return total_connections_in_group >= limits.store_limit; } - size_t getStored() const - { - std::lock_guard lock(mutex); - return total_connections_in_group; - } - - size_t getStoreLimit() const - { - std::lock_guard lock(mutex); - return limits.store_limit; - } - void atConnectionCreate() { std::lock_guard lock(mutex); @@ -359,33 +347,12 @@ private: Session::flushRequest(); } - String printAllHeaders(Poco::Net::HTTPMessage & message) const - { - String out; - out.reserve(300); - for (auto & [k, v] : message) - { - out.append(fmt::format("<{}: {}> ", k, v)); - } - return out; - } - std::ostream & sendRequest(Poco::Net::HTTPRequest & request) override { auto idle = idleTime(); std::ostream & result = Session::sendRequest(request); result.exceptions(std::ios::badbit); - // that line is for temporary debug, will be removed - LOG_INFO(log, "Send request to {} with: version {}, method {}, request no {}, keep-alive timeout={}, last usage ago: {}ms, headers: {}", - request.getVersion(), - request.getMethod(), - getTarget(), - Session::getKeepAliveRequest(), - Session::getKeepAliveTimeout().totalSeconds(), - idle.totalMilliseconds(), - printAllHeaders(request)); - request_stream = &result; request_stream_completed = false; @@ -397,22 +364,9 @@ private: std::istream & receiveResponse(Poco::Net::HTTPResponse & response) override { - int originKA = Session::getKeepAliveTimeout().totalSeconds(); - std::istream & result = Session::receiveResponse(response); result.exceptions(std::ios::badbit); - // that line is for temporary debug, will be removed - LOG_INFO(log, "Received response from {} with: version {}, code {}, request no {}, keep alive header: {}, original ka {}, last usage ago: {}ms, headers: {}", - getTarget(), - response.getVersion(), - int(response.getStatus()), - Session::getKeepAliveRequest(), - response.get(Poco::Net::HTTPMessage::CONNECTION_KEEP_ALIVE, Poco::Net::HTTPMessage::EMPTY), - originKA, - idleTime().totalMilliseconds(), - printAllHeaders(response)); - response_stream = &result; response_stream_completed = false; @@ -456,19 +410,8 @@ private: group->atConnectionDestroy(); if (!isExpired) - { if (auto lock = pool.lock()) lock->atConnectionDestroy(*this); - } - else - { - Poco::Timestamp now; - LOG_INFO(log, "Expired connection to {} with: request no {}, keep alive timeout: {}, last usage ago: {}s", - getTarget(), - Session::getKeepAliveRequest(), - Session::getKeepAliveTimeout().totalSeconds(), - idleTime().totalSeconds()); - } CurrentMetrics::sub(metrics.active_count); } @@ -519,8 +462,6 @@ private: IHTTPConnectionPoolForEndpoint::Metrics metrics; bool isExpired = false; - size_t exception_level = std::uncaught_exceptions(); - LoggerPtr log = getLogger("PooledConnection"); std::ostream * request_stream = nullptr; @@ -701,9 +642,6 @@ private: { if (connection.getKeepAliveRequest() >= connection.getKeepAliveMaxRequests()) { - LOG_INFO(getLogger("PooledConnection"), "Expired by connection number {}", - connection.getKeepAliveRequest()); - ProfileEvents::increment(getMetrics().expired, 1); return; } @@ -711,19 +649,6 @@ private: if (!connection.connected() || connection.mustReconnect() || !connection.isCompleted() || connection.buffered() || group->isStoreLimitReached()) { - Poco::Timestamp now; - LOG_INFO(getLogger("PooledConnection"), - "Reset connection to {} with: usage count {}, keep alive timeout: {}, connected {}, must recon {}, last usage ago: {}, is completed {}, store limit reached {} as {}/{}, there is exception {}", - getTarget(), - connection.getKeepAliveRequest(), - connection.getKeepAliveTimeout().totalSeconds(), - connection.connected(), - connection.mustReconnect(), - connection.idleTime().totalSeconds(), - connection.isCompleted(), - group->isStoreLimitReached(), group->getStored(), group->getStoreLimit(), - connection.exception_level - std::uncaught_exceptions()); - ProfileEvents::increment(getMetrics().reset, 1); return; } @@ -833,7 +758,6 @@ private: ConnectionGroup::Ptr storage_group = std::make_shared(HTTPConnectionGroupType::STORAGE); ConnectionGroup::Ptr http_group = std::make_shared(HTTPConnectionGroupType::HTTP); - /// If multiple mutexes are held simultaneously, /// they should be locked in this order: /// HTTPConnectionPools::mutex, then EndpointConnectionPool::mutex, then ConnectionGroup::mutex. From f47f96a84eb4c45d2c62eb3a6672ed556c5189bc Mon Sep 17 00:00:00 2001 From: Anita Hammer <166057949+anitahammer@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:32:23 +0300 Subject: [PATCH 1018/1165] Update settings.md --- .../server-configuration-parameters/settings.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 436321c8fe8..f87b6144deb 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -436,7 +436,7 @@ Default: 0 Restriction on dropping partitions. If the size of a [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table exceeds `max_partition_size_to_drop` (in bytes), you can’t drop a partition using a [DROP PARTITION](../../sql-reference/statements/alter/partition.md#drop-partitionpart) query. -This setting does not require a restart of the Clickhouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. +This setting does not require a restart of the ClickHouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. Default value: 50 GB. The value 0 means that you can drop partitions without any restrictions. @@ -518,7 +518,7 @@ Restriction on deleting tables. If the size of a [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table exceeds `max_table_size_to_drop` (in bytes), you can’t delete it using a [DROP](../../sql-reference/statements/drop.md) query or [TRUNCATE](../../sql-reference/statements/truncate.md) query. -This setting does not require a restart of the Clickhouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. +This setting does not require a restart of the ClickHouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. Default value: 50 GB. The value 0 means that you can delete all tables without any restrictions. @@ -1570,7 +1570,7 @@ Restriction on deleting tables. If the size of a [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table exceeds `max_table_size_to_drop` (in bytes), you can’t delete it using a [DROP](../../sql-reference/statements/drop.md) query or [TRUNCATE](../../sql-reference/statements/truncate.md) query. -This setting does not require a restart of the Clickhouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. +This setting does not require a restart of the ClickHouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. Default value: 50 GB. @@ -1588,7 +1588,7 @@ Restriction on dropping partitions. If the size of a [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table exceeds `max_partition_size_to_drop` (in bytes), you can’t drop a partition using a [DROP PARTITION](../../sql-reference/statements/alter/partition.md#drop-partitionpart) query. -This setting does not require a restart of the Clickhouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. +This setting does not require a restart of the ClickHouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. Default value: 50 GB. From a0bb341e82fd91e67cf1645b3c1dfecc224e7462 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 19 Mar 2024 13:40:15 +0100 Subject: [PATCH 1019/1165] Fix 02943_rmt_alter_metadata_merge_checksum_mismatch flakiness CI: https://s3.amazonaws.com/clickhouse-test-reports/60225/082a686cd1e450e18876d7a67d679c2905ec8589/fast_test.html Signed-off-by: Azat Khuzhin --- .../02943_rmt_alter_metadata_merge_checksum_mismatch.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02943_rmt_alter_metadata_merge_checksum_mismatch.sh b/tests/queries/0_stateless/02943_rmt_alter_metadata_merge_checksum_mismatch.sh index 9c0c872eb06..27950866e81 100755 --- a/tests/queries/0_stateless/02943_rmt_alter_metadata_merge_checksum_mismatch.sh +++ b/tests/queries/0_stateless/02943_rmt_alter_metadata_merge_checksum_mismatch.sh @@ -82,6 +82,8 @@ $CLICKHOUSE_CLIENT -q "optimize table $success_replica final settings optimize_t $CLICKHOUSE_CLIENT -nm --insert_keeper_fault_injection_probability=0 -q " insert into $success_replica (key) values (2); -- part all_2_2_0 + -- Avoid 'Cannot select parts for optimization: Entry for part all_2_2_0 hasn't been read from the replication log yet' + system sync replica $success_replica pull; optimize table $success_replica final settings optimize_throw_if_noop=1, alter_sync=1; -- part all_0_2_2_1 system sync replica $failed_replica pull; " From 81eda37f7f0a62cd1a4499c56a66daa7ef981827 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 5 Apr 2024 10:27:13 +0200 Subject: [PATCH 1020/1165] Print correct count --- utils/postprocess-traces/postprocess-traces.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/postprocess-traces/postprocess-traces.pl b/utils/postprocess-traces/postprocess-traces.pl index 3e50f64d864..1c198908580 100755 --- a/utils/postprocess-traces/postprocess-traces.pl +++ b/utils/postprocess-traces/postprocess-traces.pl @@ -13,9 +13,9 @@ sub process_stacktrace my $group = \$grouped_stacks; for my $frame (reverse @current_stack) { + $group = \$$group->{children}{$frame}; $$group->{count} ||= 0; ++$$group->{count}; - $group = \$$group->{children}{$frame}; } @current_stack = (); @@ -47,7 +47,7 @@ sub print_group for my $key (sort { $group->{children}{$b}{count} <=> $group->{children}{$a}{count} } keys %{$group->{children}}) { - my $count = $group->{count}; + my $count = $group->{children}{$key}{count}; print(('| ' x $level) . $count . (' ' x (5 - (length $count))) . $key . "\n"); print_group($group->{children}{$key}, $level + 1); } From 9d8f643f5b306ff02ed8e55dd776afb04e67de49 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 4 Apr 2024 20:58:35 +0000 Subject: [PATCH 1021/1165] Cleanup SSH-based authentication code --- contrib/libssh-cmake/CMakeLists.txt | 55 ++++++-------- programs/client/Client.cpp | 4 +- src/Access/Authentication.cpp | 37 ++++++--- src/Access/AuthenticationData.cpp | 13 ++-- src/Access/AuthenticationData.h | 14 +++- src/Access/Common/AuthenticationType.h | 4 +- src/Access/Credentials.h | 5 +- src/Access/User.cpp | 2 +- src/Access/UsersConfigAccessStorage.cpp | 8 +- src/CMakeLists.txt | 5 +- src/Client/Connection.cpp | 36 ++++----- src/Client/Connection.h | 12 +-- src/Client/ConnectionParameters.cpp | 11 ++- src/Client/ConnectionParameters.h | 7 +- src/Client/ConnectionPool.h | 2 +- .../{SSH/Wrappers.cpp => SSHWrapper.cpp} | 75 +++++++++---------- src/Common/{SSH/Wrappers.h => SSHWrapper.h} | 36 ++++----- src/Core/Protocol.h | 9 ++- src/Parsers/Access/ParserPublicSSHKey.cpp | 2 +- src/Server/TCPHandler.cpp | 50 ++++++------- src/Server/TCPHandler.h | 3 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- .../0_stateless/02867_create_user_ssh.sql | 12 ++- 23 files changed, 205 insertions(+), 199 deletions(-) rename src/Common/{SSH/Wrappers.cpp => SSHWrapper.cpp} (66%) rename src/Common/{SSH/Wrappers.h => SSHWrapper.h} (73%) diff --git a/contrib/libssh-cmake/CMakeLists.txt b/contrib/libssh-cmake/CMakeLists.txt index 7b589718140..ecd1fccb800 100644 --- a/contrib/libssh-cmake/CMakeLists.txt +++ b/contrib/libssh-cmake/CMakeLists.txt @@ -1,26 +1,18 @@ -option (ENABLE_SSH "Enable support for SSH keys and protocol" ${ENABLE_LIBRARIES}) +option (ENABLE_SSH "Enable support for libssh" ${ENABLE_LIBRARIES}) if (NOT ENABLE_SSH) - message(STATUS "Not using SSH") + message(STATUS "Not using libssh") return() endif() +# CMake variables needed by libssh_version.h.cmake, update them when you update libssh +set(libssh_VERSION_MAJOR 0) +set(libssh_VERSION_MINOR 9) +set(libssh_VERSION_PATCH 8) + set(LIB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libssh") set(LIB_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/libssh") -# Set CMake variables which are used in libssh_version.h.cmake -project(libssh VERSION 0.9.8 LANGUAGES C) - -set(LIBRARY_VERSION "4.8.8") -set(LIBRARY_SOVERSION "4") - -set(CMAKE_THREAD_PREFER_PTHREADS ON) -set(THREADS_PREFER_PTHREAD_FLAG ON) - -set(WITH_ZLIB OFF) -set(WITH_SYMBOL_VERSIONING OFF) -set(WITH_SERVER ON) - set(libssh_SRCS ${LIB_SOURCE_DIR}/src/agent.c ${LIB_SOURCE_DIR}/src/auth.c @@ -28,15 +20,21 @@ set(libssh_SRCS ${LIB_SOURCE_DIR}/src/bignum.c ${LIB_SOURCE_DIR}/src/buffer.c ${LIB_SOURCE_DIR}/src/callbacks.c + ${LIB_SOURCE_DIR}/src/chachapoly.c ${LIB_SOURCE_DIR}/src/channels.c ${LIB_SOURCE_DIR}/src/client.c ${LIB_SOURCE_DIR}/src/config.c + ${LIB_SOURCE_DIR}/src/config_parser.c ${LIB_SOURCE_DIR}/src/connect.c ${LIB_SOURCE_DIR}/src/connector.c ${LIB_SOURCE_DIR}/src/curve25519.c ${LIB_SOURCE_DIR}/src/dh.c ${LIB_SOURCE_DIR}/src/ecdh.c ${LIB_SOURCE_DIR}/src/error.c + ${LIB_SOURCE_DIR}/src/external/bcrypt_pbkdf.c + ${LIB_SOURCE_DIR}/src/external/blowfish.c + ${LIB_SOURCE_DIR}/src/external/chacha.c + ${LIB_SOURCE_DIR}/src/external/poly1305.c ${LIB_SOURCE_DIR}/src/getpass.c ${LIB_SOURCE_DIR}/src/init.c ${LIB_SOURCE_DIR}/src/kdf.c @@ -55,37 +53,32 @@ set(libssh_SRCS ${LIB_SOURCE_DIR}/src/pcap.c ${LIB_SOURCE_DIR}/src/pki.c ${LIB_SOURCE_DIR}/src/pki_container_openssh.c + ${LIB_SOURCE_DIR}/src/pki_ed25519_common.c ${LIB_SOURCE_DIR}/src/poll.c - ${LIB_SOURCE_DIR}/src/session.c ${LIB_SOURCE_DIR}/src/scp.c + ${LIB_SOURCE_DIR}/src/session.c ${LIB_SOURCE_DIR}/src/socket.c ${LIB_SOURCE_DIR}/src/string.c ${LIB_SOURCE_DIR}/src/threads.c - ${LIB_SOURCE_DIR}/src/wrapper.c - ${LIB_SOURCE_DIR}/src/external/bcrypt_pbkdf.c - ${LIB_SOURCE_DIR}/src/external/blowfish.c - ${LIB_SOURCE_DIR}/src/external/chacha.c - ${LIB_SOURCE_DIR}/src/external/poly1305.c - ${LIB_SOURCE_DIR}/src/chachapoly.c - ${LIB_SOURCE_DIR}/src/config_parser.c ${LIB_SOURCE_DIR}/src/token.c - ${LIB_SOURCE_DIR}/src/pki_ed25519_common.c + ${LIB_SOURCE_DIR}/src/wrapper.c + # some files of libssh/src/ are missing - why? ${LIB_SOURCE_DIR}/src/threads/noop.c ${LIB_SOURCE_DIR}/src/threads/pthread.c + # files missing - why? # LIBCRYPT specific - ${libssh_SRCS} - ${LIB_SOURCE_DIR}/src/threads/libcrypto.c - ${LIB_SOURCE_DIR}/src/pki_crypto.c + ${LIB_SOURCE_DIR}/src/dh_crypto.c ${LIB_SOURCE_DIR}/src/ecdh_crypto.c ${LIB_SOURCE_DIR}/src/libcrypto.c - ${LIB_SOURCE_DIR}/src/dh_crypto.c + ${LIB_SOURCE_DIR}/src/pki_crypto.c + ${LIB_SOURCE_DIR}/src/threads/libcrypto.c - ${LIB_SOURCE_DIR}/src/options.c - ${LIB_SOURCE_DIR}/src/server.c ${LIB_SOURCE_DIR}/src/bind.c ${LIB_SOURCE_DIR}/src/bind_config.c + ${LIB_SOURCE_DIR}/src/options.c + ${LIB_SOURCE_DIR}/src/server.c ) if (NOT (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC)) @@ -94,7 +87,7 @@ endif() configure_file(${LIB_SOURCE_DIR}/include/libssh/libssh_version.h.cmake ${LIB_BINARY_DIR}/include/libssh/libssh_version.h @ONLY) -add_library(_ssh STATIC ${libssh_SRCS}) +add_library(_ssh ${libssh_SRCS}) add_library(ch_contrib::ssh ALIAS _ssh) target_link_libraries(_ssh PRIVATE OpenSSL::Crypto) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 192f9e61891..72cad1dac07 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -934,8 +934,8 @@ void Client::addOptions(OptionsDescription & options_description) ("user,u", po::value()->default_value("default"), "user") ("password", po::value(), "password") ("ask-password", "ask-password") - ("ssh-key-file", po::value(), "File containing ssh private key needed for authentication. If not set does password authentication.") - ("ssh-key-passphrase", po::value(), "Passphrase for imported ssh key.") + ("ssh-key-file", po::value(), "File containing the SSH private key for authenticate with the server.") + ("ssh-key-passphrase", po::value(), "Passphrase for the SSH private key specified by --ssh-key-file.") ("quota_key", po::value(), "A string to differentiate quotas when the user have keyed quotas configured on server") ("max_client_network_bandwidth", po::value(), "the maximum speed of data exchange over the network for the client in bytes per second.") diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 47187d83154..bf1fe3feec3 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -4,11 +4,12 @@ #include #include #include -#include #include +#include +#include #include -#include +#include "config.h" namespace DB { @@ -74,7 +75,7 @@ namespace } #if USE_SSH - bool checkSshSignature(const std::vector & keys, std::string_view signature, std::string_view original) + bool checkSshSignature(const std::vector & keys, std::string_view signature, std::string_view original) { for (const auto & key: keys) if (key.isPublic() && key.verifySignature(signature, original)) @@ -114,7 +115,11 @@ bool Authentication::areCredentialsValid( throw Authentication::Require("ClickHouse X.509 Authentication"); case AuthenticationType::SSH_KEY: - throw Authentication::Require("Ssh Keys Authentication"); +#if USE_SSH + throw Authentication::Require("SSH Keys Authentication"); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); +#endif case AuthenticationType::MAX: break; @@ -145,7 +150,11 @@ bool Authentication::areCredentialsValid( throw Authentication::Require("ClickHouse X.509 Authentication"); case AuthenticationType::SSH_KEY: - throw Authentication::Require("Ssh Keys Authentication"); +#if USE_SSH + throw Authentication::Require("SSH Keys Authentication"); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); +#endif case AuthenticationType::MAX: break; @@ -178,7 +187,11 @@ bool Authentication::areCredentialsValid( throw Authentication::Require("ClickHouse X.509 Authentication"); case AuthenticationType::SSH_KEY: - throw Authentication::Require("Ssh Keys Authentication"); +#if USE_SSH + throw Authentication::Require("SSH Keys Authentication"); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); +#endif case AuthenticationType::BCRYPT_PASSWORD: return checkPasswordBcrypt(basic_credentials->getPassword(), auth_data.getPasswordHashBinary()); @@ -216,13 +229,18 @@ bool Authentication::areCredentialsValid( return auth_data.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName()); case AuthenticationType::SSH_KEY: - throw Authentication::Require("Ssh Keys Authentication"); +#if USE_SSH + throw Authentication::Require("SSH Keys Authentication"); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); +#endif case AuthenticationType::MAX: break; } } +#if USE_SSH if (const auto * ssh_credentials = typeid_cast(&credentials)) { switch (auth_data.getType()) @@ -243,15 +261,12 @@ bool Authentication::areCredentialsValid( throw Authentication::Require("ClickHouse X.509 Authentication"); case AuthenticationType::SSH_KEY: -#if USE_SSH return checkSshSignature(auth_data.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal()); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL"); -#endif case AuthenticationType::MAX: break; } } +#endif if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp index da90a0f5842..a4c25b438e8 100644 --- a/src/Access/AuthenticationData.cpp +++ b/src/Access/AuthenticationData.cpp @@ -105,7 +105,10 @@ bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs) return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash) && (lhs.ldap_server_name == rhs.ldap_server_name) && (lhs.kerberos_realm == rhs.kerberos_realm) && (lhs.ssl_certificate_common_names == rhs.ssl_certificate_common_names) - && (lhs.ssh_keys == rhs.ssh_keys) && (lhs.http_auth_scheme == rhs.http_auth_scheme) +#if USE_SSH + && (lhs.ssh_keys == rhs.ssh_keys) +#endif + && (lhs.http_auth_scheme == rhs.http_auth_scheme) && (lhs.http_auth_server_name == rhs.http_auth_server_name); } @@ -326,7 +329,7 @@ std::shared_ptr AuthenticationData::toAST() const break; #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); #endif } case AuthenticationType::HTTP: @@ -355,7 +358,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que { #if USE_SSH AuthenticationData auth_data(*query.type); - std::vector keys; + std::vector keys; size_t args_size = query.children.size(); for (size_t i = 0; i < args_size; ++i) @@ -366,7 +369,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que try { - keys.emplace_back(ssh::SSHKeyFactory::makePublicFromBase64(key_base64, type)); + keys.emplace_back(SSHKeyFactory::makePublicKeyFromBase64(key_base64, type)); } catch (const std::invalid_argument &) { @@ -377,7 +380,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que auth_data.setSSHKeys(std::move(keys)); return auth_data; #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); #endif } diff --git a/src/Access/AuthenticationData.h b/src/Access/AuthenticationData.h index feef4d71d66..c97e0327b56 100644 --- a/src/Access/AuthenticationData.h +++ b/src/Access/AuthenticationData.h @@ -2,14 +2,16 @@ #include #include +#include #include #include -#include #include #include #include +#include "config.h" + namespace DB { @@ -59,8 +61,10 @@ public: const boost::container::flat_set & getSSLCertificateCommonNames() const { return ssl_certificate_common_names; } void setSSLCertificateCommonNames(boost::container::flat_set common_names_); - const std::vector & getSSHKeys() const { return ssh_keys; } - void setSSHKeys(std::vector && ssh_keys_) { ssh_keys = std::forward>(ssh_keys_); } +#if USE_SSH + const std::vector & getSSHKeys() const { return ssh_keys; } + void setSSHKeys(std::vector && ssh_keys_) { ssh_keys = std::forward>(ssh_keys_); } +#endif HTTPAuthenticationScheme getHTTPAuthenticationScheme() const { return http_auth_scheme; } void setHTTPAuthenticationScheme(HTTPAuthenticationScheme scheme) { http_auth_scheme = scheme; } @@ -94,7 +98,9 @@ private: String kerberos_realm; boost::container::flat_set ssl_certificate_common_names; String salt; - std::vector ssh_keys; +#if USE_SSH + std::vector ssh_keys; +#endif /// HTTP authentication properties String http_auth_server_name; HTTPAuthenticationScheme http_auth_scheme = HTTPAuthenticationScheme::BASIC; diff --git a/src/Access/Common/AuthenticationType.h b/src/Access/Common/AuthenticationType.h index 48ace3ca00a..506c8abd3b1 100644 --- a/src/Access/Common/AuthenticationType.h +++ b/src/Access/Common/AuthenticationType.h @@ -34,8 +34,8 @@ enum class AuthenticationType /// Password is encrypted in bcrypt hash. BCRYPT_PASSWORD, - /// Server sends a random string named `challenge` which client needs to encrypt with private key. - /// The check is performed on server side by decrypting the data and comparing with the original string. + /// Server sends a random string named `challenge` to the client. The client encrypts it with its SSH private key. + /// The server decrypts the result using the SSH public key registered for the user and compares with the original string. SSH_KEY, /// Authentication through HTTP protocol diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index 77b90eaaebc..d04f8a66541 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -3,6 +3,7 @@ #include #include +#include "config.h" namespace DB { @@ -86,10 +87,11 @@ class MySQLNative41Credentials : public CredentialsWithScramble using CredentialsWithScramble::CredentialsWithScramble; }; +#if USE_SSH class SshCredentials : public Credentials { public: - explicit SshCredentials(const String& user_name_, const String& signature_, const String& original_) + SshCredentials(const String & user_name_, const String & signature_, const String & original_) : Credentials(user_name_), signature(signature_), original(original_) { is_ready = true; @@ -117,5 +119,6 @@ private: String signature; String original; }; +#endif } diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 39930c9cf76..ef5cf722113 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -31,7 +31,7 @@ void User::setName(const String & name_) throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name is empty"); if (name_ == EncodedUserInfo::USER_INTERSERVER_MARKER) throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_); - if (startsWith(name_, EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER)) + if (name_.starts_with(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_); name = name_; } diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index b4b843fc77e..e3c45eb45ae 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -10,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -214,7 +214,7 @@ namespace Poco::Util::AbstractConfiguration::Keys entries; config.keys(ssh_keys_config, entries); - std::vector keys; + std::vector keys; for (const String& entry : entries) { const auto conf_pref = ssh_keys_config + "." + entry + "."; @@ -237,7 +237,7 @@ namespace try { - keys.emplace_back(ssh::SSHKeyFactory::makePublicFromBase64(base64_key, type)); + keys.emplace_back(SSHKeyFactory::makePublicKeyFromBase64(base64_key, type)); } catch (const std::invalid_argument &) { @@ -249,7 +249,7 @@ namespace } user->auth_data.setSSHKeys(std::move(keys)); #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); #endif } else if (has_http_auth) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 73aa409e995..da17bc1f41f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -85,7 +85,6 @@ add_headers_and_sources(clickhouse_common_io Common) add_headers_and_sources(clickhouse_common_io Common/HashTable) add_headers_and_sources(clickhouse_common_io Common/Scheduler) add_headers_and_sources(clickhouse_common_io Common/Scheduler/Nodes) -add_headers_and_sources(clickhouse_common_io Common/SSH) add_headers_and_sources(clickhouse_common_io IO) add_headers_and_sources(clickhouse_common_io IO/Archives) add_headers_and_sources(clickhouse_common_io IO/S3) @@ -99,7 +98,6 @@ add_headers_and_sources(clickhouse_compression Core) #Included these specific files to avoid linking grpc add_glob(clickhouse_compression_headers Server/ServerType.h) add_glob(clickhouse_compression_sources Server/ServerType.cpp) -add_headers_and_sources(clickhouse_compression Common/SSH) add_library(clickhouse_compression ${clickhouse_compression_headers} ${clickhouse_compression_sources}) @@ -370,8 +368,7 @@ if (TARGET ch_contrib::crc32-vpmsum) endif() if (TARGET ch_contrib::ssh) - target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::ssh) - target_link_libraries(clickhouse_compression PUBLIC ch_contrib::ssh) + target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::ssh) endif() dbms_target_link_libraries(PUBLIC ch_contrib::abseil_swiss_tables) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 180942e6b83..5a1d7a2acc4 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -67,7 +67,7 @@ Connection::~Connection() = default; Connection::Connection(const String & host_, UInt16 port_, const String & default_database_, const String & user_, const String & password_, - const ssh::SSHKey & ssh_private_key_, + [[maybe_unused]] const SSHKey & ssh_private_key_, const String & quota_key_, const String & cluster_, const String & cluster_secret_, @@ -76,7 +76,9 @@ Connection::Connection(const String & host_, UInt16 port_, Protocol::Secure secure_) : host(host_), port(port_), default_database(default_database_) , user(user_), password(password_) +#if USE_SSH , ssh_private_key(ssh_private_key_) +#endif , quota_key(quota_key_) , cluster(cluster_) , cluster_secret(cluster_secret_) @@ -276,17 +278,6 @@ void Connection::disconnect() } -String Connection::packStringForSshSign(String challenge) -{ - String message; - message.append(std::to_string(DBMS_TCP_PROTOCOL_VERSION)); - message.append(default_database); - message.append(user); - message.append(challenge); - return message; -} - - void Connection::sendHello() { /** Disallow control characters in user controlled parameters @@ -334,10 +325,10 @@ void Connection::sendHello() #endif } #if USE_SSH - /// Just inform server that we will authenticate using SSH keys. else if (!ssh_private_key.isEmpty()) { - writeStringBinary(fmt::format("{}{}", EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER, user), *out); + /// Inform server that we will authenticate using SSH keys. + writeStringBinary(String(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER) + user, *out); writeStringBinary(password, *out); performHandshakeForSSHAuth(); @@ -361,9 +352,9 @@ void Connection::sendAddendum() } +#if USE_SSH void Connection::performHandshakeForSSHAuth() { -#if USE_SSH String challenge; { writeVarUInt(Protocol::Client::SSHChallengeRequest, *out); @@ -388,12 +379,23 @@ void Connection::performHandshakeForSSHAuth() } writeVarUInt(Protocol::Client::SSHChallengeResponse, *out); - String to_sign = packStringForSshSign(challenge); + + auto pack_string_for_ssh_sign = [&](String challenge_) + { + String message; + message.append(std::to_string(DBMS_TCP_PROTOCOL_VERSION)); + message.append(default_database); + message.append(user); + message.append(challenge_); + return message; + }; + + String to_sign = pack_string_for_ssh_sign(challenge); String signature = ssh_private_key.signString(to_sign); writeStringBinary(signature, *out); out->next(); -#endif } +#endif void Connection::receiveHello(const Poco::Timespan & handshake_timeout) diff --git a/src/Client/Connection.h b/src/Client/Connection.h index 5d0411027a1..2cd325afed2 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -1,10 +1,9 @@ #pragma once - #include -#include #include +#include #include #include @@ -53,7 +52,7 @@ public: Connection(const String & host_, UInt16 port_, const String & default_database_, const String & user_, const String & password_, - const ssh::SSHKey & ssh_private_key_, + const SSHKey & ssh_private_key_, const String & quota_key_, const String & cluster_, const String & cluster_secret_, @@ -170,7 +169,9 @@ private: String default_database; String user; String password; - ssh::SSHKey ssh_private_key; +#if USE_SSH + SSHKey ssh_private_key; +#endif String quota_key; /// For inter-server authorization @@ -265,9 +266,10 @@ private: void connect(const ConnectionTimeouts & timeouts); void sendHello(); - String packStringForSshSign(String challenge); +#if USE_SSH void performHandshakeForSSHAuth(); +#endif void sendAddendum(); void receiveHello(const Poco::Timespan & handshake_timeout); diff --git a/src/Client/ConnectionParameters.cpp b/src/Client/ConnectionParameters.cpp index 16911f97e84..774f3375f63 100644 --- a/src/Client/ConnectionParameters.cpp +++ b/src/Client/ConnectionParameters.cpp @@ -1,11 +1,10 @@ #include "ConnectionParameters.h" -#include + #include #include #include #include #include -#include #include #include #include @@ -88,19 +87,19 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati } else { - std::string prompt{"Enter your private key passphrase (leave empty for no passphrase): "}; + std::string prompt{"Enter your SSH private key passphrase (leave empty for no passphrase): "}; char buf[1000] = {}; if (auto * result = readpassphrase(prompt.c_str(), buf, sizeof(buf), 0)) passphrase = result; } - ssh::SSHKey key = ssh::SSHKeyFactory::makePrivateFromFile(filename, passphrase); + SSHKey key = SSHKeyFactory::makePrivateKeyFromFile(filename, passphrase); if (!key.isPrivate()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Found public key in file: {} but expected private", filename); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "File {} did not contain a private key (is it a public key?)", filename); ssh_private_key = std::move(key); #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); #endif } diff --git a/src/Client/ConnectionParameters.h b/src/Client/ConnectionParameters.h index 5f375f09c83..f23522d48b3 100644 --- a/src/Client/ConnectionParameters.h +++ b/src/Client/ConnectionParameters.h @@ -1,9 +1,10 @@ #pragma once -#include +#include #include #include -#include + +#include namespace Poco::Util { @@ -20,7 +21,7 @@ struct ConnectionParameters std::string user; std::string password; std::string quota_key; - ssh::SSHKey ssh_private_key; + SSHKey ssh_private_key; Protocol::Secure security = Protocol::Secure::Disable; Protocol::Compression compression = Protocol::Compression::Enable; ConnectionTimeouts timeouts; diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h index 574c4992d75..d35c2552461 100644 --- a/src/Client/ConnectionPool.h +++ b/src/Client/ConnectionPool.h @@ -123,7 +123,7 @@ protected: { return std::make_shared( host, port, - default_database, user, password, ssh::SSHKey(), quota_key, + default_database, user, password, SSHKey(), quota_key, cluster, cluster_secret, client_name, compression, secure); } diff --git a/src/Common/SSH/Wrappers.cpp b/src/Common/SSHWrapper.cpp similarity index 66% rename from src/Common/SSH/Wrappers.cpp rename to src/Common/SSHWrapper.cpp index a9b9f758c6e..0ed266f215c 100644 --- a/src/Common/SSH/Wrappers.cpp +++ b/src/Common/SSHWrapper.cpp @@ -1,4 +1,5 @@ -#include +#include + # if USE_SSH # include @@ -10,6 +11,14 @@ # pragma clang diagnostic pop +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LIBSSH_ERROR; +} + namespace { @@ -18,17 +27,19 @@ class SSHString public: explicit SSHString(std::string_view input) { - string = ssh_string_new(input.size()); - ssh_string_fill(string, input.data(), input.size()); + if (string = ssh_string_new(input.size()); string == nullptr) + throw Exception(ErrorCodes::LIBSSH_ERROR, "Can't create SSHString"); + if (int rc = ssh_string_fill(string, input.data(), input.size()); rc != SSH_OK) + throw Exception(ErrorCodes::LIBSSH_ERROR, "Can't create SSHString"); } - explicit SSHString(ssh_string c_other) { string = c_other; } + explicit SSHString(ssh_string other) { string = other; } ssh_string get() { return string; } String toString() { - return String(ssh_string_get_char(string), ssh_string_len(string)); + return {ssh_string_get_char(string), ssh_string_len(string)}; } ~SSHString() @@ -42,46 +53,28 @@ private: } -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LIBSSH_ERROR; -} - -namespace ssh -{ - -SSHKey SSHKeyFactory::makePrivateFromFile(String filename, String passphrase) +SSHKey SSHKeyFactory::makePrivateKeyFromFile(String filename, String passphrase) { ssh_key key; - int rc = ssh_pki_import_privkey_file(filename.c_str(), passphrase.c_str(), nullptr, nullptr, &key); - if (rc != SSH_OK) - { + if (int rc = ssh_pki_import_privkey_file(filename.c_str(), passphrase.c_str(), nullptr, nullptr, &key); rc != SSH_OK) throw Exception(ErrorCodes::LIBSSH_ERROR, "Can't import SSH private key from file"); - } return SSHKey(key); } -SSHKey SSHKeyFactory::makePublicFromFile(String filename) +SSHKey SSHKeyFactory::makePublicKeyFromFile(String filename) { ssh_key key; - int rc = ssh_pki_import_pubkey_file(filename.c_str(), &key); - if (rc != SSH_OK) + if (int rc = ssh_pki_import_pubkey_file(filename.c_str(), &key); rc != SSH_OK) throw Exception(ErrorCodes::LIBSSH_ERROR, "Can't import SSH public key from file"); - return SSHKey(key); } -SSHKey SSHKeyFactory::makePublicFromBase64(String base64_key, String type_name) +SSHKey SSHKeyFactory::makePublicKeyFromBase64(String base64_key, String type_name) { ssh_key key; auto key_type = ssh_key_type_from_name(type_name.c_str()); - int rc = ssh_pki_import_pubkey_base64(base64_key.c_str(), key_type, &key); - if (rc != SSH_OK) + if (int rc = ssh_pki_import_pubkey_base64(base64_key.c_str(), key_type, &key); rc != SSH_OK) throw Exception(ErrorCodes::LIBSSH_ERROR, "Bad SSH public key provided"); - return SSHKey(key); } @@ -90,6 +83,12 @@ SSHKey::SSHKey(const SSHKey & other) key = ssh_key_dup(other.key); } +SSHKey::SSHKey(SSHKey && other) noexcept +{ + key = other.key; + other.key = nullptr; +} + SSHKey & SSHKey::operator=(const SSHKey & other) { ssh_key_free(key); @@ -119,13 +118,11 @@ bool SSHKey::isEqual(const SSHKey & other) const String SSHKey::signString(std::string_view input) const { SSHString input_str(input); - ssh_string c_output = nullptr; - int rc = pki_sign_string(key, input_str.get(), &c_output); - if (rc != SSH_OK) + ssh_string output = nullptr; + if (int rc = pki_sign_string(key, input_str.get(), &output); rc != SSH_OK) throw Exception(ErrorCodes::LIBSSH_ERROR, "Error singing with ssh key"); - - SSHString output(c_output); - return output.toString(); + SSHString output_str(output); + return output_str.toString(); } bool SSHKey::verifySignature(std::string_view signature, std::string_view original) const @@ -149,18 +146,15 @@ namespace { struct CStringDeleter { - [[maybe_unused]] void operator()(char * ptr) const { std::free(ptr); } + void operator()(char * ptr) const { std::free(ptr); } }; } String SSHKey::getBase64() const { char * buf = nullptr; - int rc = ssh_pki_export_pubkey_base64(key, &buf); - - if (rc != SSH_OK) + if (int rc = ssh_pki_export_pubkey_base64(key, &buf); rc != SSH_OK) throw DB::Exception(DB::ErrorCodes::LIBSSH_ERROR, "Failed to export public key to base64"); - /// Create a String from cstring, which makes a copy of the first one and requires freeing memory after it /// This is to safely manage buf memory std::unique_ptr buf_ptr(buf); @@ -177,7 +171,6 @@ SSHKey::~SSHKey() ssh_key_free(key); // it's safe free from libssh } -} } #endif diff --git a/src/Common/SSH/Wrappers.h b/src/Common/SSHWrapper.h similarity index 73% rename from src/Common/SSH/Wrappers.h rename to src/Common/SSHWrapper.h index 699bba2b042..b6f0c577edc 100644 --- a/src/Common/SSH/Wrappers.h +++ b/src/Common/SSHWrapper.h @@ -1,20 +1,18 @@ #pragma once -#include -#include "config.h" -#if USE_SSH -# include -# include +#include + +#include +#include + +#include "config.h" + +#if USE_SSH using ssh_key = struct ssh_key_struct *; namespace DB { -namespace ssh -{ - -class SSHKeyFactory; - class SSHKey { public: @@ -22,11 +20,7 @@ public: ~SSHKey(); SSHKey(const SSHKey & other); - SSHKey(SSHKey && other) noexcept - { - key = other.key; - other.key = nullptr; - } + SSHKey(SSHKey && other) noexcept; SSHKey & operator=(const SSHKey & other); SSHKey & operator=(SSHKey && other) noexcept; @@ -43,7 +37,7 @@ public: String getBase64() const; String getKeyType() const; - friend SSHKeyFactory; + friend class SSHKeyFactory; private: explicit SSHKey(ssh_key key_) : key(key_) { } ssh_key key = nullptr; @@ -56,17 +50,14 @@ public: /// The check whether the path is allowed to read for ClickHouse has /// (e.g. a file is inside `user_files` directory) /// to be done outside of this functions. - static SSHKey makePrivateFromFile(String filename, String passphrase); - static SSHKey makePublicFromFile(String filename); - static SSHKey makePublicFromBase64(String base64_key, String type_name); + static SSHKey makePrivateKeyFromFile(String filename, String passphrase); + static SSHKey makePublicKeyFromFile(String filename); + static SSHKey makePublicKeyFromBase64(String base64_key, String type_name); }; -} } #else -namespace ssh -{ class SSHKey { public: @@ -74,5 +65,4 @@ public: [[ noreturn ]] bool isEmpty() { std::terminate(); } [[ noreturn ]] String signString(std::string_view) const { std::terminate(); } }; -} #endif diff --git a/src/Core/Protocol.h b/src/Core/Protocol.h index 441e22f4a16..48107154753 100644 --- a/src/Core/Protocol.h +++ b/src/Core/Protocol.h @@ -56,10 +56,11 @@ namespace DB namespace EncodedUserInfo { -/// Marker of the inter-server secret (passed in the user name) +/// Marker for the inter-server secret (passed as the user name) /// (anyway user cannot be started with a whitespace) const char USER_INTERSERVER_MARKER[] = " INTERSERVER SECRET "; -/// Marker of the SSH keys based authentication (passed in the user name) + +/// Marker for SSH-keys-based authentication (passed as the user name) const char SSH_KEY_AUTHENTICAION_MARKER[] = " SSH KEY AUTHENTICATION "; }; @@ -160,8 +161,8 @@ namespace Protocol ReadTaskResponse = 9, /// A filename to read from s3 (used in s3Cluster) MergeTreeReadTaskResponse = 10, /// Coordinator's decision with a modified set of mark ranges allowed to read - SSHChallengeRequest = 11, /// Request for SSH signature challenge - SSHChallengeResponse = 12, /// Request for SSH signature challenge + SSHChallengeRequest = 11, /// Request SSH signature challenge + SSHChallengeResponse = 12, /// Reply to SSH signature challenge MAX = SSHChallengeResponse, }; diff --git a/src/Parsers/Access/ParserPublicSSHKey.cpp b/src/Parsers/Access/ParserPublicSSHKey.cpp index bc033e25bbb..9102044900d 100644 --- a/src/Parsers/Access/ParserPublicSSHKey.cpp +++ b/src/Parsers/Access/ParserPublicSSHKey.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 5c08c697434..4e3d6ab69f6 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1371,17 +1371,6 @@ std::string formatHTTPErrorResponseWhenUserIsConnectedToWrongPort(const Poco::Ut return result; } -[[ maybe_unused ]] String createChallenge() -{ -#if USE_SSL - pcg64_fast rng(randomSeed()); - UInt64 rand = rng(); - return encodeSHA256(&rand, sizeof(rand)); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Can't generate challenge, because ClickHouse was built without OpenSSL"); -#endif -} - } std::unique_ptr TCPHandler::makeSession() @@ -1399,16 +1388,6 @@ std::unique_ptr TCPHandler::makeSession() return res; } -String TCPHandler::prepareStringForSshValidation(String username, String challenge) -{ - String output; - output.append(std::to_string(client_tcp_protocol_version)); - output.append(default_database); - output.append(username); - output.append(challenge); - return output; -} - void TCPHandler::receiveHello() { /// Receive `hello` packet. @@ -1466,11 +1445,9 @@ void TCPHandler::receiveHello() return; } - is_ssh_based_auth = startsWith(user, EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER) && password.empty(); + is_ssh_based_auth = user.starts_with(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER) && password.empty(); if (is_ssh_based_auth) - { - user.erase(0, String(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER).size()); - } + user.erase(0, std::string_view(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER).size()); session = makeSession(); const auto & client_info = session->getClientInfo(); @@ -1498,7 +1475,9 @@ void TCPHandler::receiveHello() } } } +#endif +#if USE_SSH /// Perform handshake for SSH authentication if (is_ssh_based_auth) { @@ -1512,7 +1491,14 @@ void TCPHandler::receiveHello() if (packet_type != Protocol::Client::SSHChallengeRequest) throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Server expected to receive a packet for requesting a challenge string"); - auto challenge = createChallenge(); + auto create_challenge = []() + { + pcg64_fast rng(randomSeed()); + UInt64 rand = rng(); + return encodeSHA256(&rand, sizeof(rand)); + }; + + String challenge = create_challenge(); writeVarUInt(Protocol::Server::SSHChallenge, *out); writeStringBinary(challenge, *out); out->next(); @@ -1523,7 +1509,17 @@ void TCPHandler::receiveHello() throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Server expected to receive a packet with a response for a challenge"); readStringBinary(signature, *in); - auto cred = SshCredentials(user, signature, prepareStringForSshValidation(user, challenge)); + auto prepare_string_for_ssh_validation = [&](const String & username, const String & challenge_) + { + String output; + output.append(std::to_string(client_tcp_protocol_version)); + output.append(default_database); + output.append(username); + output.append(challenge_); + return output; + }; + + auto cred = SshCredentials(user, signature, prepare_string_for_ssh_validation(user, challenge)); session->authenticate(cred, getClientAddress(client_info)); return; } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 28259d3a325..191617f1905 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -216,7 +216,7 @@ private: String default_database; - bool is_ssh_based_auth = false; + bool is_ssh_based_auth = false; /// authentication is via SSH pub-key challenge /// For inter-server secret (remote_server.*.secret) bool is_interserver_mode = false; bool is_interserver_authenticated = false; @@ -248,7 +248,6 @@ private: void extractConnectionSettingsFromContext(const ContextPtr & context); std::unique_ptr makeSession(); - String prepareStringForSshValidation(String user, String challenge); bool receiveProxyHeader(); void receiveHello(); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index c4b84a0ae8c..29ebd114b9c 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5656,7 +5656,7 @@ std::optional StorageReplicatedMergeTree::distributedWriteFromClu { auto connection = std::make_shared( node.host_name, node.port, query_context->getGlobalContext()->getCurrentDatabase(), - node.user, node.password, ssh::SSHKey(), node.quota_key, node.cluster, node.cluster_secret, + node.user, node.password, SSHKey(), node.quota_key, node.cluster, node.cluster_secret, "ParallelInsertSelectInititiator", node.compression, node.secure diff --git a/tests/queries/0_stateless/02867_create_user_ssh.sql b/tests/queries/0_stateless/02867_create_user_ssh.sql index 08236bdbcfe..3e3cb30a601 100644 --- a/tests/queries/0_stateless/02867_create_user_ssh.sql +++ b/tests/queries/0_stateless/02867_create_user_ssh.sql @@ -1,10 +1,16 @@ -- Tags: no-fasttest, no-parallel +-- Tests user authentication with SSH public keys + DROP USER IF EXISTS test_user_02867; -CREATE USER test_user_02867 IDENTIFIED WITH ssh_key BY KEY 'clickhouse' TYPE 'ssh-rsa'; -- { serverError LIBSSH_ERROR } -CREATE USER test_user_02867 IDENTIFIED WITH ssh_key BY KEY 'clickhouse' TYPE 'clickhouse'; -- { serverError LIBSSH_ERROR } -CREATE USER test_user_02867 IDENTIFIED WITH ssh_key BY KEY 'key1' TYPE 'ssh-rsa', KEY 'key2' TYPE 'ssh-rsa'; -- { serverError LIBSSH_ERROR } +-- negative tests +CREATE USER test_user_02867 IDENTIFIED WITH ssh_key BY KEY 'invalid_key' TYPE 'ssh-rsa'; -- { serverError LIBSSH_ERROR } +CREATE USER test_user_02867 IDENTIFIED WITH ssh_key BY KEY 'invalid_key' TYPE 'ssh-rsa', KEY 'invalid_key' TYPE 'ssh-rsa'; -- { serverError LIBSSH_ERROR } +CREATE USER test_user_02867 IDENTIFIED WITH ssh_key +BY KEY 'AAAAB3NzaC1yc2EAAAADAQABAAABgQCVTUso7/LQcBljfsHwyuL6fWfIvS3BaVpYB8lwf/ZylSOltBy6YlABtTU3mIb197d2DW99RcLKk174f5Zj5rUukXbV0fnufWvwd37fbb1eKM8zxBYvXs53EI5QBPZgKACIzMpYYZeJnAP0oZhUfWWtKXpy/SQ5CHiEIGD9RNYDL+uXZejMwC5r/+f2AmrATBo+Y+WJFZIvhj4uznFYvyvNTUz/YDvZCk+vwwIgiv4BpFCaZm2TeETTj6SvK567bZznLP5HXrkVbB5lhxjAkahc2w/Yjm//Fwto3xsMoJwROxJEU8L1kZ40QWPqjo7Tmr6C/hL2cKDNgWOEqrjLKQmh576s1+PfxwXpVPjLK4PHVSvuJLV88sn0iPdspLlKlDCdc7T9MqIrjJfxuhqnaoFQ7U+oBte8vkm1wGu76+WEC3iNWVAiIVZxLx9rUEsDqj3OovqfLiRsTmNLeY94p2asZjkx7rU48ZwuYN5XGafYsArPscj9Ve6RoRrof+5Q7cc=' +TYPE 'invalid_algorithm'; -- { serverError LIBSSH_ERROR } + CREATE USER test_user_02867 IDENTIFIED WITH ssh_key BY KEY 'AAAAB3NzaC1yc2EAAAADAQABAAABgQCVTUso7/LQcBljfsHwyuL6fWfIvS3BaVpYB8lwf/ZylSOltBy6YlABtTU3mIb197d2DW99RcLKk174f5Zj5rUukXbV0fnufWvwd37fbb1eKM8zxBYvXs53EI5QBPZgKACIzMpYYZeJnAP0oZhUfWWtKXpy/SQ5CHiEIGD9RNYDL+uXZejMwC5r/+f2AmrATBo+Y+WJFZIvhj4uznFYvyvNTUz/YDvZCk+vwwIgiv4BpFCaZm2TeETTj6SvK567bZznLP5HXrkVbB5lhxjAkahc2w/Yjm//Fwto3xsMoJwROxJEU8L1kZ40QWPqjo7Tmr6C/hL2cKDNgWOEqrjLKQmh576s1+PfxwXpVPjLK4PHVSvuJLV88sn0iPdspLlKlDCdc7T9MqIrjJfxuhqnaoFQ7U+oBte8vkm1wGu76+WEC3iNWVAiIVZxLx9rUEsDqj3OovqfLiRsTmNLeY94p2asZjkx7rU48ZwuYN5XGafYsArPscj9Ve6RoRrof+5Q7cc=' TYPE 'ssh-rsa'; From 0be983ee22130613c01504c328f4ec992fd728b9 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 5 Apr 2024 09:16:00 +0000 Subject: [PATCH 1022/1165] Fix test --- src/Interpreters/Context.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index faee0602f56..e4d5d895ceb 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -48,7 +48,7 @@ #include #include #include -#include +//#include #include #include #include @@ -1658,7 +1658,7 @@ void Context::addScalar(const String & name, const Block & block) throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot have scalars"); std::lock_guard lock(mutex); - scalars.emplace(name, block); + scalars[name] = block; } @@ -1668,7 +1668,7 @@ void Context::addSpecialScalar(const String & name, const Block & block) throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot have local scalars"); std::lock_guard lock(mutex); - special_scalars.emplace(name, block); + special_scalars[name] = block; } From c3aa9323677139e49552710265d7ce96f6e023de Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 5 Apr 2024 11:39:28 +0200 Subject: [PATCH 1023/1165] Uncommonly header --- src/Interpreters/Context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index e4d5d895ceb..f8a46ec30b4 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -48,7 +48,7 @@ #include #include #include -//#include +#include #include #include #include From ce1f5144177c404c955bd006f0428ee932ad49ac Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 5 Apr 2024 10:39:05 +0000 Subject: [PATCH 1024/1165] Fix optimize_uniq_to_count when only prefix of key is matched --- src/Analyzer/Passes/UniqToCountPass.cpp | 13 +++++++++++-- .../02990_optimize_uniq_to_count_alias.reference | 1 + .../02990_optimize_uniq_to_count_alias.sql | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/Analyzer/Passes/UniqToCountPass.cpp b/src/Analyzer/Passes/UniqToCountPass.cpp index d7d11e9a580..b801865c9a5 100644 --- a/src/Analyzer/Passes/UniqToCountPass.cpp +++ b/src/Analyzer/Passes/UniqToCountPass.cpp @@ -29,7 +29,8 @@ NamesAndTypes extractProjectionColumnsForGroupBy(const QueryNode * query_node) return {}; NamesAndTypes result; - for (const auto & group_by_ele : query_node->getGroupByNode()->getChildren()) + const auto & group_by_elements = query_node->getGroupByNode()->getChildren(); + for (const auto & group_by_element : group_by_elements) { const auto & projection_columns = query_node->getProjectionColumns(); const auto & projection_nodes = query_node->getProjection().getNodes(); @@ -38,10 +39,18 @@ NamesAndTypes extractProjectionColumnsForGroupBy(const QueryNode * query_node) for (size_t i = 0; i < projection_columns.size(); i++) { - if (projection_nodes[i]->isEqual(*group_by_ele)) + if (projection_nodes[i]->isEqual(*group_by_element)) + { result.push_back(projection_columns[i]); + break; + } } } + /// If some group by keys are not matched, we cannot apply optimization, + /// because prefix of group by keys may not be unique. + if (result.size() != group_by_elements.size()) + return {}; + return result; } diff --git a/tests/queries/0_stateless/02990_optimize_uniq_to_count_alias.reference b/tests/queries/0_stateless/02990_optimize_uniq_to_count_alias.reference index 6ed281c757a..e8183f05f5d 100644 --- a/tests/queries/0_stateless/02990_optimize_uniq_to_count_alias.reference +++ b/tests/queries/0_stateless/02990_optimize_uniq_to_count_alias.reference @@ -1,2 +1,3 @@ 1 1 +1 diff --git a/tests/queries/0_stateless/02990_optimize_uniq_to_count_alias.sql b/tests/queries/0_stateless/02990_optimize_uniq_to_count_alias.sql index 5ba0be39991..54d19264c45 100644 --- a/tests/queries/0_stateless/02990_optimize_uniq_to_count_alias.sql +++ b/tests/queries/0_stateless/02990_optimize_uniq_to_count_alias.sql @@ -34,4 +34,19 @@ FROM ) AS t ) SETTINGS optimize_uniq_to_count=1; +-- https://github.com/ClickHouse/ClickHouse/issues/62298 +DROP TABLE IF EXISTS users; +CREATE TABLE users +( + `id` Int64, + `name` String +) +ENGINE = ReplacingMergeTree +ORDER BY (id, name); + +INSERT INTO users VALUES (1, 'pufit'), (1, 'pufit2'), (1, 'pufit3'); + +SELECT uniqExact(id) FROM ( SELECT id FROM users WHERE id = 1 GROUP BY id, name ); + +DROP TABLE IF EXISTS users; DROP TABLE IF EXISTS tags; From 500c3fe0fcb197f7d8b2f0a6148480727015acf1 Mon Sep 17 00:00:00 2001 From: Sean Haynes Date: Fri, 5 Apr 2024 10:38:28 +0000 Subject: [PATCH 1025/1165] Fix small typo in Dictionary source loader --- src/Interpreters/ExternalLoader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 36664cbd06f..53e91971d92 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -1186,7 +1186,7 @@ private: else { auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE(log, "Supposed update time for unspecified object is {} (backoff, {} errors.", to_string(result), error_count); + LOG_TRACE(log, "Supposed update time for unspecified object is {} (backoff, {} errors)", to_string(result), error_count); return result; } } From 6428868843eb4666a3ec1defff662f673c8a5e37 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 5 Apr 2024 12:42:03 +0200 Subject: [PATCH 1026/1165] Fix build --- src/Interpreters/Cache/LRUFileCachePriority.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 9424156a9fb..5d75c9cb18c 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -259,9 +259,9 @@ bool LRUFileCachePriority::canFit( const size_t * max_elements_) const { return (max_size == 0 - || (state->current_size + size - released_size_assumption <= (max_size_ ? *max_size_ : max_size))) + || (state->current_size + size - released_size_assumption <= (max_size_ ? *max_size_ : max_size.load()))) && (max_elements == 0 - || state->current_elements_num + elements - released_elements_assumption <= (max_elements_ ? *max_elements_ : max_elements)); + || state->current_elements_num + elements - released_elements_assumption <= (max_elements_ ? *max_elements_ : max_elements.load())); } bool LRUFileCachePriority::collectCandidatesForEviction( From 07893fab631a06524658311411f53e00ef758dd3 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 5 Apr 2024 12:01:39 +0000 Subject: [PATCH 1027/1165] Revert "Fixing 02535_analyzer_group_by_use_nulls" This reverts commit 3b6ea659dfbfe25983bf1cdbdaac51ce38f6d73b. --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 29 ++++++++++------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 837d309d031..cab6dd268ea 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -6679,48 +6679,45 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR if (query_node_typed.isGroupByWithGroupingSets()) { - QueryTreeNodes nullable_group_by_keys; for (auto & grouping_sets_keys_list_node : query_node_typed.getGroupBy().getNodes()) { if (settings.enable_positional_arguments) replaceNodesWithPositionalArguments(grouping_sets_keys_list_node, query_node_typed.getProjection().getNodes(), scope); + resolveExpressionNodeList(grouping_sets_keys_list_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + // Remove redundant calls to `tuple` function. It simplifies checking if expression is an aggregation key. // It's required to support queries like: SELECT number FROM numbers(3) GROUP BY (number, number % 2) auto & group_by_list = grouping_sets_keys_list_node->as().getNodes(); expandTuplesInList(group_by_list); - - if (scope.group_by_use_nulls) - for (const auto & group_by_elem : group_by_list) - nullable_group_by_keys.push_back(group_by_elem->clone()); - - resolveExpressionNodeList(grouping_sets_keys_list_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); } - for (auto & nullable_group_by_key : nullable_group_by_keys) - scope.nullable_group_by_keys.insert(std::move(nullable_group_by_key)); + if (scope.group_by_use_nulls) + { + for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes()) + { + for (const auto & group_by_elem : grouping_set->as()->getNodes()) + scope.nullable_group_by_keys.insert(group_by_elem); + } + } } else { if (settings.enable_positional_arguments) replaceNodesWithPositionalArguments(query_node_typed.getGroupByNode(), query_node_typed.getProjection().getNodes(), scope); + resolveExpressionNodeList(query_node_typed.getGroupByNode(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + // Remove redundant calls to `tuple` function. It simplifies checking if expression is an aggregation key. // It's required to support queries like: SELECT number FROM numbers(3) GROUP BY (number, number % 2) auto & group_by_list = query_node_typed.getGroupBy().getNodes(); expandTuplesInList(group_by_list); - QueryTreeNodes nullable_group_by_keys; if (scope.group_by_use_nulls) { for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) - nullable_group_by_keys.push_back(group_by_elem->clone()); + scope.nullable_group_by_keys.insert(group_by_elem); } - - resolveExpressionNodeList(query_node_typed.getGroupByNode(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - - for (auto & nullable_group_by_key : nullable_group_by_keys) - scope.nullable_group_by_keys.insert(std::move(nullable_group_by_key)); } } From f7fdb2c4555db5530abfc2a7b10342d1e6e0217d Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 5 Apr 2024 14:23:04 +0200 Subject: [PATCH 1028/1165] More complex locking in StackTrace --- src/Common/StackTrace.cpp | 69 ++++++++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 891850ccb79..78ab43e8991 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -18,13 +18,10 @@ #include #include #include -#include #include #include #include -#include "config.h" - #include #if defined(OS_DARWIN) @@ -481,7 +478,17 @@ void StackTrace::toStringEveryLine(void ** frame_pointers_raw, size_t offset, si toStringEveryLineImpl(true, {frame_pointers, offset, size}, std::move(callback)); } -using StackTraceCache = std::map>; +struct CacheEntry +{ + std::optional stacktrace_string; + bool to_string_in_progress = false; + + std::condition_variable cv; +}; + +using CacheEntryPtr = std::shared_ptr; + +using StackTraceCache = std::map>; static StackTraceCache & cacheInstance() { @@ -493,23 +500,63 @@ static std::mutex stacktrace_cache_mutex; String toStringCached(const StackTrace::FramePointers & pointers, size_t offset, size_t size) { + const StackTraceRefTriple key{pointers, offset, size}; + /// Calculation of stack trace text is extremely slow. /// We use simple cache because otherwise the server could be overloaded by trash queries. /// Note that this cache can grow unconditionally, but practically it should be small. - std::lock_guard lock{stacktrace_cache_mutex}; - + std::unique_lock lock{stacktrace_cache_mutex}; + CacheEntryPtr cache_entry; StackTraceCache & cache = cacheInstance(); - const StackTraceRefTriple key{pointers, offset, size}; - if (auto it = cache.find(key); it != cache.end()) - return it->second; + { + cache_entry = it->second; + } else + { + auto [new_it, inserted] = cache.emplace(StackTraceTriple{pointers, offset, size}, std::make_shared()); + chassert(inserted); + cache_entry = new_it->second; + } + + if (!cache_entry->to_string_in_progress && cache_entry->stacktrace_string.has_value()) + return *cache_entry->stacktrace_string; + + if (cache_entry->to_string_in_progress) + { + cache_entry->cv.wait(lock, [&]{ return !cache_entry->to_string_in_progress; }); + + if (cache_entry->stacktrace_string.has_value()) + return *cache_entry->stacktrace_string; + } + + cache_entry->to_string_in_progress = true; + + lock.unlock(); + + String stacktrace_string; + try { DB::WriteBufferFromOwnString out; toStringEveryLineImpl(false, key, [&](std::string_view str) { out << str << '\n'; }); - - return cache.emplace(StackTraceTriple{pointers, offset, size}, out.str()).first->second; + stacktrace_string = out.str(); } + catch (...) + { + lock.lock(); + cache_entry->to_string_in_progress = false; + lock.unlock(); + cache_entry->cv.notify_one(); + throw; + } + + lock.lock(); + cache_entry->to_string_in_progress = false; + cache_entry->stacktrace_string = stacktrace_string; + lock.unlock(); + + cache_entry->cv.notify_all(); + return stacktrace_string; } std::string StackTrace::toString() const From 8b2c719aa665b0af7d2190c29ad842d9b0aa1fdd Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 5 Apr 2024 12:57:02 +0000 Subject: [PATCH 1029/1165] Do not build multithread insert pipeline for tables without support --- src/Interpreters/InterpreterInsertQuery.cpp | 3 ++- src/QueryPipeline/Pipe.cpp | 2 -- .../03035_max_insert_threads_support.reference | 1 + .../03035_max_insert_threads_support.sh | 14 ++++++++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/03035_max_insert_threads_support.reference create mode 100755 tests/queries/0_stateless/03035_max_insert_threads_support.sh diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index fc58f7b5098..d2eda928d8c 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -522,7 +522,8 @@ BlockIO InterpreterInsertQuery::execute() auto views = DatabaseCatalog::instance().getDependentViews(table_id); /// It breaks some views-related tests and we have dedicated `parallel_view_processing` for views, so let's just skip them. - const bool resize_to_max_insert_threads = !table->isView() && views.empty(); + /// Also it doesn't make sense to reshuffle data if storage doesn't support parallel inserts. + const bool resize_to_max_insert_threads = !table->isView() && views.empty() && table->supportsParallelInsert(); pre_streams_size = resize_to_max_insert_threads ? settings.max_insert_threads : std::min(settings.max_insert_threads, pipeline.getNumStreams()); diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index 8050c7cc671..34602ecccee 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -13,8 +13,6 @@ #include #include -#include - namespace DB { diff --git a/tests/queries/0_stateless/03035_max_insert_threads_support.reference b/tests/queries/0_stateless/03035_max_insert_threads_support.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03035_max_insert_threads_support.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03035_max_insert_threads_support.sh b/tests/queries/0_stateless/03035_max_insert_threads_support.sh new file mode 100755 index 00000000000..1e6bfb414d8 --- /dev/null +++ b/tests/queries/0_stateless/03035_max_insert_threads_support.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DATA_FILE="data_$CLICKHOUSE_TEST_UNIQUE_NAME.csv" + +$CLICKHOUSE_CLIENT --max_insert_threads=4 --query=" + EXPLAIN PIPELINE INSERT INTO FUNCTION file('$DATA_FILE') SELECT * FROM numbers_mt(1000000) ORDER BY number DESC +" | grep -o MaterializingTransform | wc -l + +DATA_FILE_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path from file('$DATA_FILE', 'One')") +rm $DATA_FILE_PATH From e53ba4fa9db4646ee3a0c193594379b33043bcf2 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 5 Apr 2024 13:32:07 +0000 Subject: [PATCH 1030/1165] Analyzer: Fix PREWHERE with lambda functions --- src/Planner/CollectTableExpressionData.cpp | 4 +++- .../0_stateless/03036_prewhere_lambda_function.reference | 2 ++ .../0_stateless/03036_prewhere_lambda_function.sql | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03036_prewhere_lambda_function.reference create mode 100644 tests/queries/0_stateless/03036_prewhere_lambda_function.sql diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp index 385381f1355..27b5909c13b 100644 --- a/src/Planner/CollectTableExpressionData.cpp +++ b/src/Planner/CollectTableExpressionData.cpp @@ -235,7 +235,9 @@ public: static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) { auto child_node_type = child_node->getNodeType(); - return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); + return child_node_type != QueryTreeNodeType::QUERY && + child_node_type != QueryTreeNodeType::UNION && + child_node_type != QueryTreeNodeType::LAMBDA; } private: diff --git a/tests/queries/0_stateless/03036_prewhere_lambda_function.reference b/tests/queries/0_stateless/03036_prewhere_lambda_function.reference new file mode 100644 index 00000000000..470e4427d96 --- /dev/null +++ b/tests/queries/0_stateless/03036_prewhere_lambda_function.reference @@ -0,0 +1,2 @@ +[4,5,6] +[4,5,6] diff --git a/tests/queries/0_stateless/03036_prewhere_lambda_function.sql b/tests/queries/0_stateless/03036_prewhere_lambda_function.sql new file mode 100644 index 00000000000..7a5da7ed689 --- /dev/null +++ b/tests/queries/0_stateless/03036_prewhere_lambda_function.sql @@ -0,0 +1,8 @@ +DROP TABLE IF EXISTS t; +CREATE TABLE t (A Array(Int64)) Engine = MergeTree ORDER BY tuple(); +INSERT INTO t VALUES ([1,2,3]), ([4,5,6]), ([7,8,9]); + +SELECT * FROM t PREWHERE arrayExists(x -> x = 5, A); +SELECT * FROM t PREWHERE arrayExists(lamdba(tuple(x), x = 5), A); + +DROP TABLE t; From 54ceb3d32a7bb490ba7f202a511607f0ea21ae5b Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 5 Apr 2024 12:47:00 +0000 Subject: [PATCH 1031/1165] add some comments --- src/Processors/QueryPlan/PartsSplitter.cpp | 2 ++ .../test_final_bug_with_pk_columns_loading/test.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index ec51875587e..64af48dd53c 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -128,6 +128,8 @@ class IndexAccess public: explicit IndexAccess(const RangesInDataParts & parts_) : parts(parts_) { + /// Some suffix of index columns might not be loaded (see `primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns`) + /// and we need to use the same set of index columns across all parts. for (const auto & part : parts) loaded_columns = std::min(loaded_columns, part.data_part->getIndex().size()); } diff --git a/tests/integration/test_final_bug_with_pk_columns_loading/test.py b/tests/integration/test_final_bug_with_pk_columns_loading/test.py index e710b9942dc..61559913e05 100644 --- a/tests/integration/test_final_bug_with_pk_columns_loading/test.py +++ b/tests/integration/test_final_bug_with_pk_columns_loading/test.py @@ -19,18 +19,24 @@ def start_cluster(): cluster.shutdown() -def test_simple_query_after_restart(start_cluster): +def test_simple_query_after_index_reload(start_cluster): node.query( """ create table t(a UInt32, b UInt32) engine=MergeTree order by (a, b) settings index_granularity=1; + -- for this part the first columns is useless, so we have to use both insert into t select 42, number from numbers_mt(100); + + -- for this part the first columns is enough insert into t select number, number from numbers_mt(100); """ ) + # force reloading index node.restart_clickhouse() + # the bug happened when we used (a, b) index values for one part and only (a) for another in PartsSplitter. even a simple count query is enough, + # because some granules were assinged to wrong layers and hence not returned from the reading step (because they were filtered out by `FilterSortedStreamByRange`) assert ( int( node.query( From 0f4efdaa4788dc5fd9e4ee96ca611eb35d63a29a Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 5 Apr 2024 14:48:39 +0000 Subject: [PATCH 1032/1165] remove case from 03036_prewhere_lambda_function --- .../queries/0_stateless/03036_prewhere_lambda_function.reference | 1 - tests/queries/0_stateless/03036_prewhere_lambda_function.sql | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/queries/0_stateless/03036_prewhere_lambda_function.reference b/tests/queries/0_stateless/03036_prewhere_lambda_function.reference index 470e4427d96..2599763b762 100644 --- a/tests/queries/0_stateless/03036_prewhere_lambda_function.reference +++ b/tests/queries/0_stateless/03036_prewhere_lambda_function.reference @@ -1,2 +1 @@ [4,5,6] -[4,5,6] diff --git a/tests/queries/0_stateless/03036_prewhere_lambda_function.sql b/tests/queries/0_stateless/03036_prewhere_lambda_function.sql index 7a5da7ed689..8b9ebb775a3 100644 --- a/tests/queries/0_stateless/03036_prewhere_lambda_function.sql +++ b/tests/queries/0_stateless/03036_prewhere_lambda_function.sql @@ -3,6 +3,5 @@ CREATE TABLE t (A Array(Int64)) Engine = MergeTree ORDER BY tuple(); INSERT INTO t VALUES ([1,2,3]), ([4,5,6]), ([7,8,9]); SELECT * FROM t PREWHERE arrayExists(x -> x = 5, A); -SELECT * FROM t PREWHERE arrayExists(lamdba(tuple(x), x = 5), A); DROP TABLE t; From 39d706ba9f0c8e7f8c8d757e215f639f7d510fe2 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 5 Apr 2024 14:45:51 +0000 Subject: [PATCH 1033/1165] rework test --- .../__init__.py | 0 .../test.py | 53 ------------------- ...s_splitter_bug_and_index_loading.reference | 1 + ...3_parts_splitter_bug_and_index_loading.sql | 17 ++++++ 4 files changed, 18 insertions(+), 53 deletions(-) delete mode 100644 tests/integration/test_final_bug_with_pk_columns_loading/__init__.py delete mode 100644 tests/integration/test_final_bug_with_pk_columns_loading/test.py create mode 100644 tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.reference create mode 100644 tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.sql diff --git a/tests/integration/test_final_bug_with_pk_columns_loading/__init__.py b/tests/integration/test_final_bug_with_pk_columns_loading/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_final_bug_with_pk_columns_loading/test.py b/tests/integration/test_final_bug_with_pk_columns_loading/test.py deleted file mode 100644 index 61559913e05..00000000000 --- a/tests/integration/test_final_bug_with_pk_columns_loading/test.py +++ /dev/null @@ -1,53 +0,0 @@ -import pytest -import logging - -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", stay_alive=True) - - -@pytest.fixture(scope="module") -def start_cluster(): - try: - logging.info("Starting cluster...") - cluster.start() - logging.info("Cluster started") - - yield cluster - finally: - cluster.shutdown() - - -def test_simple_query_after_index_reload(start_cluster): - node.query( - """ - create table t(a UInt32, b UInt32) engine=MergeTree order by (a, b) settings index_granularity=1; - - -- for this part the first columns is useless, so we have to use both - insert into t select 42, number from numbers_mt(100); - - -- for this part the first columns is enough - insert into t select number, number from numbers_mt(100); - """ - ) - - # force reloading index - node.restart_clickhouse() - - # the bug happened when we used (a, b) index values for one part and only (a) for another in PartsSplitter. even a simple count query is enough, - # because some granules were assinged to wrong layers and hence not returned from the reading step (because they were filtered out by `FilterSortedStreamByRange`) - assert ( - int( - node.query( - "select count() from t where not ignore(*)", - settings={ - "max_threads": 4, - "merge_tree_min_bytes_for_concurrent_read": 1, - "merge_tree_min_rows_for_concurrent_read": 1, - "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability": 1, - }, - ) - ) - == 200 - ) diff --git a/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.reference b/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.reference new file mode 100644 index 00000000000..08839f6bb29 --- /dev/null +++ b/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.reference @@ -0,0 +1 @@ +200 diff --git a/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.sql b/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.sql new file mode 100644 index 00000000000..541ac67fd24 --- /dev/null +++ b/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.sql @@ -0,0 +1,17 @@ +create table t(a UInt32, b UInt32) engine=MergeTree order by (a, b) settings index_granularity=1; + +-- for this part the first columns is useless, so we have to use both +insert into t select 42, number from numbers_mt(100); + +-- for this part the first columns is enough +insert into t select number, number from numbers_mt(100); + +-- force reloading index +detach table t; +attach table t; + +set merge_tree_min_bytes_for_concurrent_read=1, merge_tree_min_rows_for_concurrent_read=1, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=1.0, max_threads=4; + +-- the bug happened when we used (a, b) index values for one part and only (a) for another in PartsSplitter. even a simple count query is enough, +-- because some granules were assinged to wrong layers and hence not returned from the reading step (because they were filtered out by `FilterSortedStreamByRange`) +select count() from t where not ignore(*); From b2bcfaf344047f629879143d6bb4efa00c22f7cb Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Fri, 5 Apr 2024 17:18:22 +0200 Subject: [PATCH 1034/1165] Reduce log levels for ReadWriteBufferFromHTTP retries --- src/IO/ReadWriteBufferFromHTTP.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp index c99b08d0c9d..303ffb744b5 100644 --- a/src/IO/ReadWriteBufferFromHTTP.cpp +++ b/src/IO/ReadWriteBufferFromHTTP.cpp @@ -345,7 +345,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, if (last_attempt || !is_retriable) { if (!mute_logging) - LOG_ERROR(log, + LOG_DEBUG(log, "Failed to make request to '{}'{}. " "Error: '{}'. " "Failed at try {}/{}.", @@ -361,7 +361,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, on_retry(); if (!mute_logging) - LOG_INFO(log, + LOG_TRACE(log, "Failed to make request to '{}'{}. " "Error: {}. " "Failed at try {}/{}. " From 7d50bb8c4bf2f7d51d5aece0bb42a1ca4e8afac0 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 5 Apr 2024 12:33:43 -0300 Subject: [PATCH 1035/1165] fix ut once again --- src/IO/tests/gtest_s3_uri.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index 175550acccc..0ec28f80072 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -96,7 +96,7 @@ const TestCase TestCases[] = { false}, // Zonal {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com/root/nested/file.txt"), - "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com", + "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com", "root", "nested/file.txt", "", From 6e413223c2560007bab6422117e4d284c3aefdd4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 5 Apr 2024 09:47:24 +0200 Subject: [PATCH 1036/1165] Use DETACHED_DIR_NAME everywhere Signed-off-by: Azat Khuzhin --- .../MergeTree/DataPartStorageOnDiskBase.cpp | 10 ++++--- src/Storages/MergeTree/DataPartsExchange.cpp | 4 +-- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 +-- src/Storages/MergeTree/MergeTreeData.cpp | 27 +++++++++---------- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 16 +++++------ 6 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index 18e4c87b298..052e3ba4b74 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include namespace DB @@ -64,7 +66,7 @@ std::optional DataPartStorageOnDiskBase::getRelativePathForPrefix(Logger auto full_relative_path = fs::path(root_path); if (detached) - full_relative_path /= "detached"; + full_relative_path /= MergeTreeData::DETACHED_DIR_NAME; std::optional original_checksums_content; std::optional original_files_list; @@ -109,7 +111,7 @@ bool DataPartStorageOnDiskBase::looksLikeBrokenDetachedPartHasTheSameContent(con if (!exists("checksums.txt")) return false; - auto storage_from_detached = create(volume, fs::path(root_path) / "detached", detached_part_path, /*initialize=*/ true); + auto storage_from_detached = create(volume, fs::path(root_path) / MergeTreeData::DETACHED_DIR_NAME, detached_part_path, /*initialize=*/ true); if (!storage_from_detached->exists("checksums.txt")) return false; @@ -490,7 +492,7 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::freeze( auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); /// Do not initialize storage in case of DETACH because part may be broken. - bool to_detached = dir_path.starts_with("detached/"); + bool to_detached = dir_path.starts_with(std::string_view((fs::path(MergeTreeData::DETACHED_DIR_NAME) / "").string())); return create(single_disk_volume, to, dir_path, /*initialize=*/ !to_detached && !params.external_transaction); } @@ -618,7 +620,7 @@ void DataPartStorageOnDiskBase::remove( if (part_dir_without_slash.has_parent_path()) { auto parent_path = part_dir_without_slash.parent_path(); - if (parent_path == "detached") + if (parent_path == MergeTreeData::DETACHED_DIR_NAME) throw Exception( ErrorCodes::LOGICAL_ERROR, "Trying to remove detached part {} with path {} in remove function. It shouldn't happen", diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 91444d76a52..cf7889c0aee 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,6 @@ #include #include #include -#include #include @@ -803,7 +803,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( throw Exception(ErrorCodes::LOGICAL_ERROR, "`tmp_prefix` and `part_name` cannot be empty or contain '.' or '/' characters."); auto part_dir = tmp_prefix + part_name; - auto part_relative_path = data.getRelativeDataPath() + String(to_detached ? "detached/" : ""); + auto part_relative_path = data.getRelativeDataPath() + String(to_detached ? MergeTreeData::DETACHED_DIR_NAME : ""); auto volume = std::make_shared("volume_" + part_name, disk); /// Create temporary part storage to write sent files. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 8da46b39801..441437855ab 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1844,7 +1844,7 @@ try } catch (...) { - if (startsWith(new_relative_path, "detached/")) + if (startsWith(new_relative_path, fs::path(MergeTreeData::DETACHED_DIR_NAME) / "")) { // Don't throw when the destination is to the detached folder. It might be able to // recover in some cases, such as fetching parts into multi-disks while some of the @@ -1957,7 +1957,7 @@ std::optional IMergeTreeDataPart::getRelativePathForDetachedPart(const S DetachedPartInfo::DETACH_REASONS.end(), prefix) != DetachedPartInfo::DETACH_REASONS.end()); if (auto path = getRelativePathForPrefix(prefix, /* detached */ true, broken)) - return "detached/" + *path; + return fs::path(MergeTreeData::DETACHED_DIR_NAME) / *path; return {}; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 461d9a31eaa..dc15b8ab940 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -262,7 +262,7 @@ void MergeTreeData::initializeDirectoriesAndFormatVersion(const std::string & re if (need_create_directories) { disk->createDirectories(relative_data_path); - disk->createDirectories(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME); + disk->createDirectories(fs::path(relative_data_path) / DETACHED_DIR_NAME); } if (disk->exists(format_version_path)) @@ -1713,7 +1713,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks, std::optionalname(), "tmp") || it->name() == MergeTreeData::FORMAT_VERSION_FILE_NAME - || it->name() == MergeTreeData::DETACHED_DIR_NAME) + || it->name() == DETACHED_DIR_NAME) continue; if (auto part_info = MergeTreePartInfo::tryParsePartName(it->name(), format_version)) @@ -2796,7 +2796,7 @@ void MergeTreeData::dropAllData() && settings_ptr->allow_remote_fs_zero_copy_replication; try { - bool keep_shared = removeDetachedPart(part.disk, fs::path(relative_data_path) / "detached" / part.dir_name / "", part.dir_name); + bool keep_shared = removeDetachedPart(part.disk, fs::path(relative_data_path) / DETACHED_DIR_NAME / part.dir_name / "", part.dir_name); LOG_DEBUG(log, "Dropped detached part {}, keep shared data: {}", part.dir_name, keep_shared); } catch (...) @@ -2879,8 +2879,8 @@ void MergeTreeData::dropIfEmpty() if (disk->isBroken()) continue; /// Non recursive, exception is thrown if there are more files. - disk->removeFileIfExists(fs::path(relative_data_path) / MergeTreeData::FORMAT_VERSION_FILE_NAME); - disk->removeDirectory(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME); + disk->removeFileIfExists(fs::path(relative_data_path) / FORMAT_VERSION_FILE_NAME); + disk->removeDirectory(fs::path(relative_data_path) / DETACHED_DIR_NAME); disk->removeDirectory(relative_data_path); } } @@ -3443,7 +3443,7 @@ void MergeTreeData::changeSettings( { auto disk = new_storage_policy->getDiskByName(disk_name); disk->createDirectories(relative_data_path); - disk->createDirectories(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME); + disk->createDirectories(fs::path(relative_data_path) / DETACHED_DIR_NAME); } /// FIXME how would that be done while reloading configuration??? @@ -6037,7 +6037,7 @@ DetachedPartsInfo MergeTreeData::getDetachedParts() const for (const auto & disk : getDisks()) { - String detached_path = fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME; + String detached_path = fs::path(relative_data_path) / DETACHED_DIR_NAME; /// Note: we don't care about TOCTOU issue here. if (disk->exists(detached_path)) @@ -6063,7 +6063,7 @@ void MergeTreeData::validateDetachedPartName(const String & name) void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr local_context) { - PartsTemporaryRename renamed_parts(*this, "detached/"); + PartsTemporaryRename renamed_parts(*this, DETACHED_DIR_NAME); if (part) { @@ -6088,7 +6088,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr for (auto & [old_name, new_name, disk] : renamed_parts.old_and_new_names) { - bool keep_shared = removeDetachedPart(disk, fs::path(relative_data_path) / "detached" / new_name / "", old_name); + bool keep_shared = removeDetachedPart(disk, fs::path(relative_data_path) / DETACHED_DIR_NAME / new_name / "", old_name); LOG_DEBUG(log, "Dropped detached part {}, keep shared data: {}", old_name, keep_shared); old_name.clear(); } @@ -6097,14 +6097,14 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const ASTPtr & partition, bool attach_part, ContextPtr local_context, PartsTemporaryRename & renamed_parts) { - const String source_dir = "detached/"; + const fs::path source_dir = DETACHED_DIR_NAME; /// Let's compose a list of parts that should be added. if (attach_part) { const String part_id = partition->as().value.safeGet(); validateDetachedPartName(part_id); - if (temporary_parts.contains(String(DETACHED_DIR_NAME) + "/" + part_id)) + if (temporary_parts.contains(source_dir / part_id)) { LOG_WARNING(log, "Will not try to attach part {} because its directory is temporary, " "probably it's being detached right now", part_id); @@ -6181,7 +6181,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const LOG_DEBUG(log, "Checking part {}", new_name); auto single_disk_volume = std::make_shared("volume_" + old_name, disk); - auto part = getDataPartBuilder(old_name, single_disk_volume, source_dir + new_name) + auto part = getDataPartBuilder(old_name, single_disk_volume, source_dir / new_name) .withPartFormatFromDisk() .build(); @@ -7212,11 +7212,10 @@ String MergeTreeData::getFullPathOnDisk(const DiskPtr & disk) const DiskPtr MergeTreeData::tryGetDiskForDetachedPart(const String & part_name) const { - String additional_path = "detached/"; const auto disks = getStoragePolicy()->getDisks(); for (const DiskPtr & disk : disks) - if (disk->exists(fs::path(relative_data_path) / additional_path / part_name)) + if (disk->exists(fs::path(relative_data_path) / DETACHED_DIR_NAME / part_name)) return disk; return nullptr; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index c9f451b6bb1..6861b615cd6 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2024,7 +2024,7 @@ PartitionCommandsResultInfo StorageMergeTree::attachPartition( bool attach_part, ContextPtr local_context) { PartitionCommandsResultInfo results; - PartsTemporaryRename renamed_parts(*this, "detached/"); + PartsTemporaryRename renamed_parts(*this, DETACHED_DIR_NAME); MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, local_context, renamed_parts); for (size_t i = 0; i < loaded_parts.size(); ++i) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6ab56ba141c..73354e71e71 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1983,7 +1983,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo for (const DiskPtr & disk : getStoragePolicy()->getDisks()) { - for (const auto it = disk->iterateDirectory(fs::path(relative_data_path) / "detached/"); it->isValid(); it->next()) + for (const auto it = disk->iterateDirectory(fs::path(relative_data_path) / DETACHED_DIR_NAME); it->isValid(); it->next()) { const auto part_info = MergeTreePartInfo::tryParsePartName(it->name(), format_version); @@ -1993,7 +1993,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo const auto part_old_name = part_info->getPartNameV1(); const auto volume = std::make_shared("volume_" + part_old_name, disk); - auto part = getDataPartBuilder(entry.new_part_name, volume, fs::path("detached") / part_old_name) + auto part = getDataPartBuilder(entry.new_part_name, volume, fs::path(DETACHED_DIR_NAME) / part_old_name) .withPartFormatFromDisk() .build(); @@ -2440,7 +2440,7 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) { String part_dir = part_to_detach->getDataPartStorage().getPartDirectory(); LOG_INFO(log, "Detaching {}", part_dir); - auto holder = getTemporaryPartDirectoryHolder(String(DETACHED_DIR_NAME) + "/" + part_dir); + auto holder = getTemporaryPartDirectoryHolder(fs::path(DETACHED_DIR_NAME) / part_dir); part_to_detach->makeCloneInDetached("", metadata_snapshot, /*disk_transaction*/ {}); } } @@ -2967,7 +2967,7 @@ void StorageReplicatedMergeTree::executeClonePartFromShard(const LogEntry & entr part = get_part(); // The fetched part is valuable and should not be cleaned like a temp part. part->is_temp = false; - part->renameTo("detached/" + entry.new_part_name, true); + part->renameTo(fs::path(DETACHED_DIR_NAME) / entry.new_part_name, true); LOG_INFO(log, "Cloned part {} to detached directory", part->name); } @@ -4987,7 +4987,7 @@ bool StorageReplicatedMergeTree::fetchPart( { // The fetched part is valuable and should not be cleaned like a temp part. part->is_temp = false; - part->renameTo(fs::path("detached") / part_name, true); + part->renameTo(fs::path(DETACHED_DIR_NAME) / part_name, true); } } catch (const Exception & e) @@ -6547,7 +6547,7 @@ PartitionCommandsResultInfo StorageReplicatedMergeTree::attachPartition( assertNotReadonly(); PartitionCommandsResultInfo results; - PartsTemporaryRename renamed_parts(*this, "detached/"); + PartsTemporaryRename renamed_parts(*this, DETACHED_DIR_NAME); MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, query_context, renamed_parts); /// TODO Allow to use quorum here. @@ -9986,7 +9986,7 @@ bool StorageReplicatedMergeTree::checkIfDetachedPartExists(const String & part_n { fs::directory_iterator dir_end; for (const std::string & path : getDataPaths()) - for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it) + for (fs::directory_iterator dir_it{fs::path(path) / DETACHED_DIR_NAME}; dir_it != dir_end; ++dir_it) if (dir_it->path().filename().string() == part_name) return true; return false; @@ -9999,7 +9999,7 @@ bool StorageReplicatedMergeTree::checkIfDetachedPartitionExists(const String & p for (const std::string & path : getDataPaths()) { - for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it) + for (fs::directory_iterator dir_it{fs::path(path) / DETACHED_DIR_NAME}; dir_it != dir_end; ++dir_it) { const String file_name = dir_it->path().filename().string(); auto part_info = MergeTreePartInfo::tryParsePartName(file_name, format_version); From b2c9cb0653f6d4857a9ea1eb98904c0b4d1d7526 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 5 Apr 2024 18:19:47 +0200 Subject: [PATCH 1037/1165] Fix flaky tests --- .../03049_unknown_identifier_materialized_column.sql | 4 ---- tests/queries/0_stateless/03068_analyzer_distributed_join.sql | 2 ++ .../queries/0_stateless/03084_analyzer_join_column_alias.sql | 4 ++-- .../03088_analyzer_ambiguous_column_multi_call.sql | 2 ++ 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql index a1c858a329c..276e4845831 100644 --- a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql +++ b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql @@ -1,8 +1,6 @@ -- https://github.com/ClickHouse/ClickHouse/issues/54317 SET allow_experimental_analyzer=1; DROP DATABASE IF EXISTS 03049_database; -DROP TABLE IF EXISTS 03049_database.l; -DROP TABLE IF EXISTS 03049_database.r; CREATE DATABASE 03049_database; USE 03049_database; @@ -14,5 +12,3 @@ select * from l left join r on l.y = r.y where r.ty >= 2019; select * from 03049_database.l left join 03049_database.r on l.y = r.y where r.ty >= 2019; DROP DATABASE IF EXISTS 03049_database; -DROP TABLE IF EXISTS 03049_database.l; -DROP TABLE IF EXISTS 03049_database.r; diff --git a/tests/queries/0_stateless/03068_analyzer_distributed_join.sql b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql index 82f58e9a750..61b1199dc44 100644 --- a/tests/queries/0_stateless/03068_analyzer_distributed_join.sql +++ b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql @@ -1,4 +1,6 @@ -- https://github.com/ClickHouse/ClickHouse/issues/6571 +-- Tag: no-replicated-database + SET allow_experimental_analyzer=1; CREATE TABLE LINEITEM_shard ON CLUSTER test_shard_localhost ( diff --git a/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql b/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql index 8337c0ce987..930726898b5 100644 --- a/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql +++ b/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql @@ -1,13 +1,13 @@ -- https://github.com/ClickHouse/ClickHouse/issues/47432 SET allow_experimental_analyzer=1; -create or replace table t1 +create table t1 engine = MergeTree() order by tuple() as select 1 as user_id, 2 as level; -create or replace table t2 +create table t2 engine = MergeTree() order by tuple() as diff --git a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql index 09425d2e503..4ca5005fa1d 100644 --- a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql +++ b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql @@ -1,5 +1,7 @@ -- https://github.com/ClickHouse/ClickHouse/issues/61014 SET allow_experimental_analyzer=1; + +DROP DATABASE IF EXISTS test_03088; create database test_03088; create table test_03088.a (i int) engine = Log(); From d1c42668bddc9a8c99dfd0bd8f0d340b60da4569 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 5 Apr 2024 16:22:57 +0000 Subject: [PATCH 1038/1165] Another attempt. --- src/Analyzer/ArrayJoinNode.cpp | 4 +-- src/Analyzer/ArrayJoinNode.h | 4 +-- src/Analyzer/ColumnNode.cpp | 18 ++++++++----- src/Analyzer/ColumnNode.h | 4 +-- src/Analyzer/ColumnTransformers.cpp | 12 ++++----- src/Analyzer/ColumnTransformers.h | 12 ++++----- src/Analyzer/ConstantNode.cpp | 26 ++++++++++++++----- src/Analyzer/ConstantNode.h | 9 +++---- src/Analyzer/ConstantValue.h | 10 ------- src/Analyzer/FunctionNode.cpp | 10 +++++-- src/Analyzer/FunctionNode.h | 4 +-- src/Analyzer/HashUtils.h | 24 ++++++++--------- src/Analyzer/IQueryTreeNode.cpp | 4 +-- src/Analyzer/IQueryTreeNode.h | 9 ++++--- src/Analyzer/IdentifierNode.cpp | 4 +-- src/Analyzer/IdentifierNode.h | 4 +-- src/Analyzer/InterpolateNode.cpp | 4 +-- src/Analyzer/InterpolateNode.h | 4 +-- src/Analyzer/JoinNode.cpp | 4 +-- src/Analyzer/JoinNode.h | 4 +-- src/Analyzer/LambdaNode.cpp | 4 +-- src/Analyzer/LambdaNode.h | 4 +-- src/Analyzer/ListNode.cpp | 4 +-- src/Analyzer/ListNode.h | 4 +-- src/Analyzer/MatcherNode.cpp | 4 +-- src/Analyzer/MatcherNode.h | 4 +-- src/Analyzer/Passes/QueryAnalysisPass.cpp | 21 ++++++++------- src/Analyzer/QueryNode.cpp | 4 +-- src/Analyzer/QueryNode.h | 4 +-- src/Analyzer/SortNode.cpp | 4 +-- src/Analyzer/SortNode.h | 4 +-- src/Analyzer/TableFunctionNode.cpp | 4 +-- src/Analyzer/TableFunctionNode.h | 4 +-- src/Analyzer/TableNode.cpp | 4 +-- src/Analyzer/TableNode.h | 4 +-- src/Analyzer/UnionNode.cpp | 4 +-- src/Analyzer/UnionNode.h | 4 +-- src/Analyzer/WindowNode.cpp | 4 +-- src/Analyzer/WindowNode.h | 4 +-- src/Analyzer/tests/gtest_query_tree_node.cpp | 4 +-- ...up_by_use_nulls_analyzer_crashes.reference | 2 ++ ...23_group_by_use_nulls_analyzer_crashes.sql | 2 ++ 42 files changed, 146 insertions(+), 129 deletions(-) diff --git a/src/Analyzer/ArrayJoinNode.cpp b/src/Analyzer/ArrayJoinNode.cpp index ee6bd80150d..e817a893af4 100644 --- a/src/Analyzer/ArrayJoinNode.cpp +++ b/src/Analyzer/ArrayJoinNode.cpp @@ -33,13 +33,13 @@ void ArrayJoinNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_stat getJoinExpressionsNode()->dumpTreeImpl(buffer, format_state, indent + 4); } -bool ArrayJoinNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool ArrayJoinNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); return is_left == rhs_typed.is_left; } -void ArrayJoinNode::updateTreeHashImpl(HashState & state) const +void ArrayJoinNode::updateTreeHashImpl(HashState & state, CompareOptions) const { state.update(is_left); } diff --git a/src/Analyzer/ArrayJoinNode.h b/src/Analyzer/ArrayJoinNode.h index 89cb0b7b8c1..1772e2b3ca0 100644 --- a/src/Analyzer/ArrayJoinNode.h +++ b/src/Analyzer/ArrayJoinNode.h @@ -93,9 +93,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & state) const override; + void updateTreeHashImpl(HashState & state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/ColumnNode.cpp b/src/Analyzer/ColumnNode.cpp index 3d9f5d1640e..b8d2613871d 100644 --- a/src/Analyzer/ColumnNode.cpp +++ b/src/Analyzer/ColumnNode.cpp @@ -70,20 +70,26 @@ void ColumnNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & state, size_t } } -bool ColumnNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool ColumnNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions compare_options) const { const auto & rhs_typed = assert_cast(rhs); - return column == rhs_typed.column; + if (column.name != rhs_typed.column.name) + return false; + + return !compare_options.compare_types || column.type->equals(*rhs_typed.column.type); } -void ColumnNode::updateTreeHashImpl(HashState & hash_state) const +void ColumnNode::updateTreeHashImpl(HashState & hash_state, CompareOptions compare_options) const { hash_state.update(column.name.size()); hash_state.update(column.name); - const auto & column_type_name = column.type->getName(); - hash_state.update(column_type_name.size()); - hash_state.update(column_type_name); + if (compare_options.compare_types) + { + const auto & column_type_name = column.type->getName(); + hash_state.update(column_type_name.size()); + hash_state.update(column_type_name); + } } QueryTreeNodePtr ColumnNode::cloneImpl() const diff --git a/src/Analyzer/ColumnNode.h b/src/Analyzer/ColumnNode.h index 46e7c8eb500..f6fac5ce7f9 100644 --- a/src/Analyzer/ColumnNode.h +++ b/src/Analyzer/ColumnNode.h @@ -131,9 +131,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & hash_state) const override; + void updateTreeHashImpl(HashState & hash_state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/ColumnTransformers.cpp b/src/Analyzer/ColumnTransformers.cpp index 3a6b9e3b291..356344c1aec 100644 --- a/src/Analyzer/ColumnTransformers.cpp +++ b/src/Analyzer/ColumnTransformers.cpp @@ -74,13 +74,13 @@ void ApplyColumnTransformerNode::dumpTreeImpl(WriteBuffer & buffer, FormatState expression_node->dumpTreeImpl(buffer, format_state, indent + 4); } -bool ApplyColumnTransformerNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool ApplyColumnTransformerNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); return apply_transformer_type == rhs_typed.apply_transformer_type; } -void ApplyColumnTransformerNode::updateTreeHashImpl(IQueryTreeNode::HashState & hash_state) const +void ApplyColumnTransformerNode::updateTreeHashImpl(IQueryTreeNode::HashState & hash_state, CompareOptions) const { hash_state.update(static_cast(getTransformerType())); hash_state.update(static_cast(getApplyTransformerType())); @@ -178,7 +178,7 @@ void ExceptColumnTransformerNode::dumpTreeImpl(WriteBuffer & buffer, FormatState } } -bool ExceptColumnTransformerNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool ExceptColumnTransformerNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); if (except_transformer_type != rhs_typed.except_transformer_type || @@ -198,7 +198,7 @@ bool ExceptColumnTransformerNode::isEqualImpl(const IQueryTreeNode & rhs) const return column_matcher->pattern() == rhs_column_matcher->pattern(); } -void ExceptColumnTransformerNode::updateTreeHashImpl(IQueryTreeNode::HashState & hash_state) const +void ExceptColumnTransformerNode::updateTreeHashImpl(IQueryTreeNode::HashState & hash_state, CompareOptions) const { hash_state.update(static_cast(getTransformerType())); hash_state.update(static_cast(getExceptTransformerType())); @@ -302,13 +302,13 @@ void ReplaceColumnTransformerNode::dumpTreeImpl(WriteBuffer & buffer, FormatStat } } -bool ReplaceColumnTransformerNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool ReplaceColumnTransformerNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); return is_strict == rhs_typed.is_strict && replacements_names == rhs_typed.replacements_names; } -void ReplaceColumnTransformerNode::updateTreeHashImpl(IQueryTreeNode::HashState & hash_state) const +void ReplaceColumnTransformerNode::updateTreeHashImpl(IQueryTreeNode::HashState & hash_state, CompareOptions) const { hash_state.update(static_cast(getTransformerType())); diff --git a/src/Analyzer/ColumnTransformers.h b/src/Analyzer/ColumnTransformers.h index 8fa8e28f194..9ae1f14575b 100644 --- a/src/Analyzer/ColumnTransformers.h +++ b/src/Analyzer/ColumnTransformers.h @@ -137,9 +137,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(IQueryTreeNode::HashState & hash_state) const override; + void updateTreeHashImpl(IQueryTreeNode::HashState & hash_state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; @@ -214,9 +214,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(IQueryTreeNode::HashState & hash_state) const override; + void updateTreeHashImpl(IQueryTreeNode::HashState & hash_state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; @@ -290,9 +290,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(IQueryTreeNode::HashState & hash_state) const override; + void updateTreeHashImpl(IQueryTreeNode::HashState & hash_state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/ConstantNode.cpp b/src/Analyzer/ConstantNode.cpp index e26500a9886..46c1f7fb1ed 100644 --- a/src/Analyzer/ConstantNode.cpp +++ b/src/Analyzer/ConstantNode.cpp @@ -126,17 +126,29 @@ void ConstantNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state } } -bool ConstantNode::isEqualImpl(const IQueryTreeNode & rhs) const +void ConstantNode::convertToNullable() { - const auto & rhs_typed = assert_cast(rhs); - return *constant_value == *rhs_typed.constant_value && value_string == rhs_typed.value_string; + constant_value = std::make_shared(constant_value->getValue(), makeNullableSafe(constant_value->getType())); } -void ConstantNode::updateTreeHashImpl(HashState & hash_state) const +bool ConstantNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions compare_options) const { - auto type_name = constant_value->getType()->getName(); - hash_state.update(type_name.size()); - hash_state.update(type_name); + const auto & rhs_typed = assert_cast(rhs); + + if (value_string != rhs_typed.value_string || constant_value->getValue() != rhs_typed.constant_value->getValue()) + return false; + + return !compare_options.compare_types || constant_value->getType()->equals(*rhs_typed.constant_value->getType()); +} + +void ConstantNode::updateTreeHashImpl(HashState & hash_state, CompareOptions compare_options) const +{ + if (compare_options.compare_types) + { + auto type_name = constant_value->getType()->getName(); + hash_state.update(type_name.size()); + hash_state.update(type_name); + } hash_state.update(value_string.size()); hash_state.update(value_string); diff --git a/src/Analyzer/ConstantNode.h b/src/Analyzer/ConstantNode.h index 98a8eb78277..0c88862b879 100644 --- a/src/Analyzer/ConstantNode.h +++ b/src/Analyzer/ConstantNode.h @@ -87,17 +87,14 @@ public: mask_id = id; } - void convertToNullable() override - { - constant_value = std::make_shared(constant_value->getValue(), makeNullableSafe(constant_value->getType())); - } + void convertToNullable() override; void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions compare_options) const override; - void updateTreeHashImpl(HashState & hash_state) const override; + void updateTreeHashImpl(HashState & hash_state, CompareOptions compare_options) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/ConstantValue.h b/src/Analyzer/ConstantValue.h index a9e2ffd9e65..335072b9286 100644 --- a/src/Analyzer/ConstantValue.h +++ b/src/Analyzer/ConstantValue.h @@ -34,14 +34,4 @@ private: DataTypePtr data_type; }; -inline bool operator==(const ConstantValue & lhs, const ConstantValue & rhs) -{ - return lhs.getValue() == rhs.getValue() && lhs.getType()->equals(*rhs.getType()); -} - -inline bool operator!=(const ConstantValue & lhs, const ConstantValue & rhs) -{ - return !(lhs == rhs); -} - } diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp index e902ac2274e..f13842cf67c 100644 --- a/src/Analyzer/FunctionNode.cpp +++ b/src/Analyzer/FunctionNode.cpp @@ -142,7 +142,7 @@ void FunctionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state } } -bool FunctionNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool FunctionNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions compare_options) const { const auto & rhs_typed = assert_cast(rhs); if (function_name != rhs_typed.function_name || isAggregateFunction() != rhs_typed.isAggregateFunction() @@ -150,6 +150,9 @@ bool FunctionNode::isEqualImpl(const IQueryTreeNode & rhs) const || nulls_action != rhs_typed.nulls_action) return false; + if (!compare_options.compare_types) + return true; + if (isResolved() != rhs_typed.isResolved()) return false; if (!isResolved()) @@ -168,7 +171,7 @@ bool FunctionNode::isEqualImpl(const IQueryTreeNode & rhs) const return true; } -void FunctionNode::updateTreeHashImpl(HashState & hash_state) const +void FunctionNode::updateTreeHashImpl(HashState & hash_state, CompareOptions compare_options) const { hash_state.update(function_name.size()); hash_state.update(function_name); @@ -177,6 +180,9 @@ void FunctionNode::updateTreeHashImpl(HashState & hash_state) const hash_state.update(isWindowFunction()); hash_state.update(nulls_action); + if (!compare_options.compare_types) + return; + if (!isResolved()) return; diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index 0ff3e689632..8d14b7eeb0d 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -208,9 +208,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions compare_options) const override; - void updateTreeHashImpl(HashState & hash_state) const override; + void updateTreeHashImpl(HashState & hash_state, CompareOptions compare_options) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/HashUtils.h b/src/Analyzer/HashUtils.h index eb6aac88fe9..80f59c1eaaa 100644 --- a/src/Analyzer/HashUtils.h +++ b/src/Analyzer/HashUtils.h @@ -11,37 +11,37 @@ namespace DB * Example of usage: * std::unordered_map map; */ -template +template struct QueryTreeNodeWithHash { QueryTreeNodeWithHash(QueryTreeNodePtrType node_) /// NOLINT : node(std::move(node_)) - , hash(node->getTreeHash({.compare_aliases = compare_aliases})) + , hash(node->getTreeHash({.compare_aliases = compare_aliases, .compare_types = compare_types})) {} QueryTreeNodePtrType node = nullptr; CityHash_v1_0_2::uint128 hash; }; -template -inline bool operator==(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) +template +inline bool operator==(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) { - return lhs.hash == rhs.hash && lhs.node->isEqual(*rhs.node, {.compare_aliases = compare_aliases}); + return lhs.hash == rhs.hash && lhs.node->isEqual(*rhs.node, {.compare_aliases = compare_aliases, .compare_types = compare_types}); } -template -inline bool operator!=(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) +template +inline bool operator!=(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) { return !(lhs == rhs); } using QueryTreeNodePtrWithHash = QueryTreeNodeWithHash; -using QueryTreeNodePtrWithHashWithoutAlias = QueryTreeNodeWithHash; +using QueryTreeNodePtrWithHashIgnoreTypes = QueryTreeNodeWithHash; using QueryTreeNodeRawPtrWithHash = QueryTreeNodeWithHash; using QueryTreeNodeConstRawPtrWithHash = QueryTreeNodeWithHash; using QueryTreeNodePtrWithHashSet = std::unordered_set; -using QueryTreeNodePtrWithHashWithoutAliasSet = std::unordered_set; +using QueryTreeNodePtrWithHashIgnoreTypesSet = std::unordered_set; using QueryTreeNodeConstRawPtrWithHashSet = std::unordered_set; template @@ -52,10 +52,10 @@ using QueryTreeNodeConstRawPtrWithHashMap = std::unordered_map -struct std::hash> +template +struct std::hash> { - size_t operator()(const DB::QueryTreeNodeWithHash & node_with_hash) const + size_t operator()(const DB::QueryTreeNodeWithHash & node_with_hash) const { return node_with_hash.hash.low64; } diff --git a/src/Analyzer/IQueryTreeNode.cpp b/src/Analyzer/IQueryTreeNode.cpp index 7815b93c3ac..cd085babf38 100644 --- a/src/Analyzer/IQueryTreeNode.cpp +++ b/src/Analyzer/IQueryTreeNode.cpp @@ -107,7 +107,7 @@ bool IQueryTreeNode::isEqual(const IQueryTreeNode & rhs, CompareOptions compare_ } if (lhs_node_to_compare->getNodeType() != rhs_node_to_compare->getNodeType() || - !lhs_node_to_compare->isEqualImpl(*rhs_node_to_compare)) + !lhs_node_to_compare->isEqualImpl(*rhs_node_to_compare, compare_options)) return false; if (compare_options.compare_aliases && lhs_node_to_compare->alias != rhs_node_to_compare->alias) @@ -207,7 +207,7 @@ IQueryTreeNode::Hash IQueryTreeNode::getTreeHash(CompareOptions compare_options) hash_state.update(node_to_process->alias); } - node_to_process->updateTreeHashImpl(hash_state); + node_to_process->updateTreeHashImpl(hash_state, compare_options); hash_state.update(node_to_process->children.size()); diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index 92e34616c4d..fc2cb2c53f6 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -97,6 +97,7 @@ public: struct CompareOptions { bool compare_aliases = true; + bool compare_types = true; }; /** Is tree equal to other tree with node root. @@ -104,7 +105,7 @@ public: * With default compare options aliases of query tree nodes are compared during isEqual call. * Original ASTs of query tree nodes are not compared during isEqual call. */ - bool isEqual(const IQueryTreeNode & rhs, CompareOptions compare_options = { .compare_aliases = true }) const; + bool isEqual(const IQueryTreeNode & rhs, CompareOptions compare_options = { .compare_aliases = true, .compare_types = true }) const; using Hash = CityHash_v1_0_2::uint128; using HashState = SipHash; @@ -114,7 +115,7 @@ public: * Alias of query tree node is part of query tree hash. * Original AST is not part of query tree hash. */ - Hash getTreeHash(CompareOptions compare_options = { .compare_aliases = true }) const; + Hash getTreeHash(CompareOptions compare_options = { .compare_aliases = true, .compare_types = true }) const; /// Get a deep copy of the query tree QueryTreeNodePtr clone() const; @@ -264,12 +265,12 @@ protected: /** Subclass must compare its internal state with rhs node internal state and do not compare children or weak pointers to other * query tree nodes. */ - virtual bool isEqualImpl(const IQueryTreeNode & rhs) const = 0; + virtual bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions compare_options) const = 0; /** Subclass must update tree hash with its internal state and do not update tree hash for children or weak pointers to other * query tree nodes. */ - virtual void updateTreeHashImpl(HashState & hash_state) const = 0; + virtual void updateTreeHashImpl(HashState & hash_state, CompareOptions compare_options) const = 0; /** Subclass must clone its internal state and do not clone children or weak pointers to other * query tree nodes. diff --git a/src/Analyzer/IdentifierNode.cpp b/src/Analyzer/IdentifierNode.cpp index 88b3daacb12..181e75a57fd 100644 --- a/src/Analyzer/IdentifierNode.cpp +++ b/src/Analyzer/IdentifierNode.cpp @@ -38,13 +38,13 @@ void IdentifierNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_sta } } -bool IdentifierNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool IdentifierNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); return identifier == rhs_typed.identifier && table_expression_modifiers == rhs_typed.table_expression_modifiers; } -void IdentifierNode::updateTreeHashImpl(HashState & state) const +void IdentifierNode::updateTreeHashImpl(HashState & state, CompareOptions) const { const auto & identifier_name = identifier.getFullName(); state.update(identifier_name.size()); diff --git a/src/Analyzer/IdentifierNode.h b/src/Analyzer/IdentifierNode.h index 872bb14d512..1b07f0b3765 100644 --- a/src/Analyzer/IdentifierNode.h +++ b/src/Analyzer/IdentifierNode.h @@ -53,9 +53,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & state) const override; + void updateTreeHashImpl(HashState & state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/InterpolateNode.cpp b/src/Analyzer/InterpolateNode.cpp index d78993c7b85..e4f7e22b803 100644 --- a/src/Analyzer/InterpolateNode.cpp +++ b/src/Analyzer/InterpolateNode.cpp @@ -28,13 +28,13 @@ void InterpolateNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_st getInterpolateExpression()->dumpTreeImpl(buffer, format_state, indent + 4); } -bool InterpolateNode::isEqualImpl(const IQueryTreeNode &) const +bool InterpolateNode::isEqualImpl(const IQueryTreeNode &, CompareOptions) const { /// No state in interpolate node return true; } -void InterpolateNode::updateTreeHashImpl(HashState &) const +void InterpolateNode::updateTreeHashImpl(HashState &, CompareOptions) const { /// No state in interpolate node } diff --git a/src/Analyzer/InterpolateNode.h b/src/Analyzer/InterpolateNode.h index c45800ebaaf..9269d3924f5 100644 --- a/src/Analyzer/InterpolateNode.h +++ b/src/Analyzer/InterpolateNode.h @@ -53,9 +53,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & hash_state) const override; + void updateTreeHashImpl(HashState & hash_state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/JoinNode.cpp b/src/Analyzer/JoinNode.cpp index 9b61c8b19d0..1a440ad8abc 100644 --- a/src/Analyzer/JoinNode.cpp +++ b/src/Analyzer/JoinNode.cpp @@ -81,13 +81,13 @@ void JoinNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, si } } -bool JoinNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool JoinNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); return locality == rhs_typed.locality && strictness == rhs_typed.strictness && kind == rhs_typed.kind; } -void JoinNode::updateTreeHashImpl(HashState & state) const +void JoinNode::updateTreeHashImpl(HashState & state, CompareOptions) const { state.update(locality); state.update(strictness); diff --git a/src/Analyzer/JoinNode.h b/src/Analyzer/JoinNode.h index 4f071e03856..734162d9546 100644 --- a/src/Analyzer/JoinNode.h +++ b/src/Analyzer/JoinNode.h @@ -142,9 +142,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & state) const override; + void updateTreeHashImpl(HashState & state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/LambdaNode.cpp b/src/Analyzer/LambdaNode.cpp index 4be4d69c190..bca2616d85a 100644 --- a/src/Analyzer/LambdaNode.cpp +++ b/src/Analyzer/LambdaNode.cpp @@ -46,13 +46,13 @@ void LambdaNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, getExpression()->dumpTreeImpl(buffer, format_state, indent + 4); } -bool LambdaNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool LambdaNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); return argument_names == rhs_typed.argument_names; } -void LambdaNode::updateTreeHashImpl(HashState & state) const +void LambdaNode::updateTreeHashImpl(HashState & state, CompareOptions) const { state.update(argument_names.size()); for (const auto & argument_name : argument_names) diff --git a/src/Analyzer/LambdaNode.h b/src/Analyzer/LambdaNode.h index ea44a7e8187..0b2882125f0 100644 --- a/src/Analyzer/LambdaNode.h +++ b/src/Analyzer/LambdaNode.h @@ -97,9 +97,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & state) const override; + void updateTreeHashImpl(HashState & state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/ListNode.cpp b/src/Analyzer/ListNode.cpp index 799c471d685..217cd6cefa3 100644 --- a/src/Analyzer/ListNode.cpp +++ b/src/Analyzer/ListNode.cpp @@ -38,13 +38,13 @@ void ListNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, si } } -bool ListNode::isEqualImpl(const IQueryTreeNode &) const +bool ListNode::isEqualImpl(const IQueryTreeNode &, CompareOptions) const { /// No state return true; } -void ListNode::updateTreeHashImpl(HashState &) const +void ListNode::updateTreeHashImpl(HashState &, CompareOptions) const { /// No state } diff --git a/src/Analyzer/ListNode.h b/src/Analyzer/ListNode.h index 5b1abc36ae9..379919f190f 100644 --- a/src/Analyzer/ListNode.h +++ b/src/Analyzer/ListNode.h @@ -51,9 +51,9 @@ public: const_iterator end() const { return children.end(); } protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState &) const override; + void updateTreeHashImpl(HashState &, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index f573b83e538..341c4b8eec7 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -160,7 +160,7 @@ void MatcherNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, } } -bool MatcherNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool MatcherNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); if (matcher_type != rhs_typed.matcher_type || @@ -181,7 +181,7 @@ bool MatcherNode::isEqualImpl(const IQueryTreeNode & rhs) const return columns_matcher->pattern() == rhs_columns_matcher->pattern(); } -void MatcherNode::updateTreeHashImpl(HashState & hash_state) const +void MatcherNode::updateTreeHashImpl(HashState & hash_state, CompareOptions) const { hash_state.update(static_cast(matcher_type)); diff --git a/src/Analyzer/MatcherNode.h b/src/Analyzer/MatcherNode.h index d6f077e224b..a7ec7d984c6 100644 --- a/src/Analyzer/MatcherNode.h +++ b/src/Analyzer/MatcherNode.h @@ -135,9 +135,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & hash_state) const override; + void updateTreeHashImpl(HashState & hash_state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index cab6dd268ea..4d862639e15 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -776,7 +776,7 @@ struct IdentifierResolveScope /// Table expression node to data std::unordered_map table_expression_node_to_data; - QueryTreeNodePtrWithHashWithoutAliasSet nullable_group_by_keys; + QueryTreeNodePtrWithHashIgnoreTypesSet nullable_group_by_keys; /// Here we count the number of nullable GROUP BY keys we met resolving expression. /// E.g. for a query `SELECT tuple(tuple(number)) FROM numbers(10) GROUP BY (number, tuple(number)) with cube` /// both `number` and `tuple(number)` would be in nullable_group_by_keys. @@ -6155,12 +6155,6 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id return resolved_expression_it->second; } - bool is_nullable_group_by_key = scope.nullable_group_by_keys.contains(node); - if (is_nullable_group_by_key) - ++scope.found_nullable_group_by_key_in_scope; - - SCOPE_EXIT(scope.found_nullable_group_by_key_in_scope -= is_nullable_group_by_key); - String node_alias = node->getAlias(); ProjectionNames result_projection_names; @@ -6452,10 +6446,14 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id validateTreeSize(node, scope.context->getSettingsRef().max_expanded_ast_elements, node_to_tree_size); - if (is_nullable_group_by_key && scope.found_nullable_group_by_key_in_scope == 1 && !scope.expressions_in_resolve_process_stack.hasAggregateFunction()) + if (!scope.expressions_in_resolve_process_stack.hasAggregateFunction()) { - node = node->clone(); - node->convertToNullable(); + auto it = scope.nullable_group_by_keys.find(node); + if (it != scope.nullable_group_by_keys.end()) + { + node = it->node->clone(); + node->convertToNullable(); + } } /** Update aliases after expression node was resolved. @@ -8028,6 +8026,9 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.hasGroupBy()) resolveGroupByNode(query_node_typed, scope); + if (scope.group_by_use_nulls) + resolved_expressions.clear(); + if (query_node_typed.hasHaving()) resolveExpressionNode(query_node_typed.getHaving(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); diff --git a/src/Analyzer/QueryNode.cpp b/src/Analyzer/QueryNode.cpp index bc7a29247e4..f1361c328db 100644 --- a/src/Analyzer/QueryNode.cpp +++ b/src/Analyzer/QueryNode.cpp @@ -247,7 +247,7 @@ void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s } } -bool QueryNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool QueryNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); @@ -266,7 +266,7 @@ bool QueryNode::isEqualImpl(const IQueryTreeNode & rhs) const settings_changes == rhs_typed.settings_changes; } -void QueryNode::updateTreeHashImpl(HashState & state) const +void QueryNode::updateTreeHashImpl(HashState & state, CompareOptions) const { state.update(is_subquery); state.update(is_cte); diff --git a/src/Analyzer/QueryNode.h b/src/Analyzer/QueryNode.h index 6f9067908dd..af187df72a8 100644 --- a/src/Analyzer/QueryNode.h +++ b/src/Analyzer/QueryNode.h @@ -589,9 +589,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState &) const override; + void updateTreeHashImpl(HashState &, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/SortNode.cpp b/src/Analyzer/SortNode.cpp index 8e9913af442..795a639deea 100644 --- a/src/Analyzer/SortNode.cpp +++ b/src/Analyzer/SortNode.cpp @@ -71,7 +71,7 @@ void SortNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, si } } -bool SortNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool SortNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); if (sort_direction != rhs_typed.sort_direction || @@ -89,7 +89,7 @@ bool SortNode::isEqualImpl(const IQueryTreeNode & rhs) const return collator->getLocale() == rhs_typed.collator->getLocale(); } -void SortNode::updateTreeHashImpl(HashState & hash_state) const +void SortNode::updateTreeHashImpl(HashState & hash_state, CompareOptions) const { hash_state.update(sort_direction); /// use some determined value if `nulls_sort_direction` is `nullopt` diff --git a/src/Analyzer/SortNode.h b/src/Analyzer/SortNode.h index b860fd19a90..4d1f6f7c0f0 100644 --- a/src/Analyzer/SortNode.h +++ b/src/Analyzer/SortNode.h @@ -131,9 +131,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & hash_state) const override; + void updateTreeHashImpl(HashState & hash_state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/TableFunctionNode.cpp b/src/Analyzer/TableFunctionNode.cpp index e5158a06373..87d2fdcffb5 100644 --- a/src/Analyzer/TableFunctionNode.cpp +++ b/src/Analyzer/TableFunctionNode.cpp @@ -82,7 +82,7 @@ void TableFunctionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_ } } -bool TableFunctionNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool TableFunctionNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); if (table_function_name != rhs_typed.table_function_name) @@ -97,7 +97,7 @@ bool TableFunctionNode::isEqualImpl(const IQueryTreeNode & rhs) const return table_expression_modifiers == rhs_typed.table_expression_modifiers; } -void TableFunctionNode::updateTreeHashImpl(HashState & state) const +void TableFunctionNode::updateTreeHashImpl(HashState & state, CompareOptions) const { state.update(table_function_name.size()); state.update(table_function_name); diff --git a/src/Analyzer/TableFunctionNode.h b/src/Analyzer/TableFunctionNode.h index 69237ac8416..98121ef95c5 100644 --- a/src/Analyzer/TableFunctionNode.h +++ b/src/Analyzer/TableFunctionNode.h @@ -155,9 +155,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & state) const override; + void updateTreeHashImpl(HashState & state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/TableNode.cpp b/src/Analyzer/TableNode.cpp index f899c1ae6fe..daf5db08551 100644 --- a/src/Analyzer/TableNode.cpp +++ b/src/Analyzer/TableNode.cpp @@ -52,14 +52,14 @@ void TableNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s } } -bool TableNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool TableNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); return storage_id == rhs_typed.storage_id && table_expression_modifiers == rhs_typed.table_expression_modifiers && temporary_table_name == rhs_typed.temporary_table_name; } -void TableNode::updateTreeHashImpl(HashState & state) const +void TableNode::updateTreeHashImpl(HashState & state, CompareOptions) const { if (!temporary_table_name.empty()) { diff --git a/src/Analyzer/TableNode.h b/src/Analyzer/TableNode.h index b0bf91fa01b..2d66167acd1 100644 --- a/src/Analyzer/TableNode.h +++ b/src/Analyzer/TableNode.h @@ -100,9 +100,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & state) const override; + void updateTreeHashImpl(HashState & state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/UnionNode.cpp b/src/Analyzer/UnionNode.cpp index c6003126554..9bc2a197d9a 100644 --- a/src/Analyzer/UnionNode.cpp +++ b/src/Analyzer/UnionNode.cpp @@ -145,7 +145,7 @@ void UnionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s getQueriesNode()->dumpTreeImpl(buffer, format_state, indent + 4); } -bool UnionNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool UnionNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); @@ -153,7 +153,7 @@ bool UnionNode::isEqualImpl(const IQueryTreeNode & rhs) const union_mode == rhs_typed.union_mode; } -void UnionNode::updateTreeHashImpl(HashState & state) const +void UnionNode::updateTreeHashImpl(HashState & state, CompareOptions) const { state.update(is_subquery); state.update(is_cte); diff --git a/src/Analyzer/UnionNode.h b/src/Analyzer/UnionNode.h index 7686b73f5e0..189951f6375 100644 --- a/src/Analyzer/UnionNode.h +++ b/src/Analyzer/UnionNode.h @@ -143,9 +143,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState &) const override; + void updateTreeHashImpl(HashState &, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/WindowNode.cpp b/src/Analyzer/WindowNode.cpp index 0fbe7c51bc7..39a89ca6828 100644 --- a/src/Analyzer/WindowNode.cpp +++ b/src/Analyzer/WindowNode.cpp @@ -80,14 +80,14 @@ void WindowNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, } } -bool WindowNode::isEqualImpl(const IQueryTreeNode & rhs) const +bool WindowNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const { const auto & rhs_typed = assert_cast(rhs); return window_frame == rhs_typed.window_frame && parent_window_name == rhs_typed.parent_window_name; } -void WindowNode::updateTreeHashImpl(HashState & hash_state) const +void WindowNode::updateTreeHashImpl(HashState & hash_state, CompareOptions) const { hash_state.update(window_frame.is_default); hash_state.update(window_frame.type); diff --git a/src/Analyzer/WindowNode.h b/src/Analyzer/WindowNode.h index 30e1128b93c..febbc02bedc 100644 --- a/src/Analyzer/WindowNode.h +++ b/src/Analyzer/WindowNode.h @@ -169,9 +169,9 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: - bool isEqualImpl(const IQueryTreeNode & rhs) const override; + bool isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const override; - void updateTreeHashImpl(HashState & hash_state) const override; + void updateTreeHashImpl(HashState & hash_state, CompareOptions) const override; QueryTreeNodePtr cloneImpl() const override; diff --git a/src/Analyzer/tests/gtest_query_tree_node.cpp b/src/Analyzer/tests/gtest_query_tree_node.cpp index cf1ce78e423..01556c9f921 100644 --- a/src/Analyzer/tests/gtest_query_tree_node.cpp +++ b/src/Analyzer/tests/gtest_query_tree_node.cpp @@ -22,12 +22,12 @@ public: { } - bool isEqualImpl(const IQueryTreeNode &) const override + bool isEqualImpl(const IQueryTreeNode &, CompareOptions) const override { return true; } - void updateTreeHashImpl(HashState &) const override + void updateTreeHashImpl(HashState &, CompareOptions) const override { } diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference index 4081b82a8f5..4243abb1a1e 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference @@ -60,3 +60,5 @@ (7) (8) (9) +a b +a b diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql index 28042fc5b44..20cf47e4feb 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql @@ -7,3 +7,5 @@ select tuple(array(number)) as x FROM numbers(10) GROUP BY number, array(number) SELECT tuple(number) AS x FROM numbers(10) GROUP BY GROUPING SETS (number) order by x; SELECT ignore(toFixedString('Lambda as function parameter', 28), toNullable(28), ignore(8)), sum(marks) FROM system.parts GROUP BY GROUPING SETS ((2)) FORMAT Null settings optimize_injective_functions_in_group_by=1, optimize_group_by_function_keys=1, group_by_use_nulls=1; -- { serverError ILLEGAL_AGGREGATION } + +SELECT toLowCardinality(materialize('a' AS key)), 'b' AS value GROUP BY key WITH CUBE SETTINGS group_by_use_nulls = 1; From eb9690016a06ad544e0d819a44ad772b1d0cd2cc Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 5 Apr 2024 18:37:46 +0200 Subject: [PATCH 1039/1165] Fix assertion --- src/Interpreters/Cache/EvictionCandidates.h | 6 ---- src/Interpreters/Cache/FileCache.cpp | 21 +++++------- src/Interpreters/Cache/IFileCachePriority.h | 3 +- .../Cache/LRUFileCachePriority.cpp | 6 ++-- src/Interpreters/Cache/LRUFileCachePriority.h | 3 +- .../Cache/SLRUFileCachePriority.cpp | 33 +++++++++++-------- .../Cache/SLRUFileCachePriority.h | 3 +- 7 files changed, 37 insertions(+), 38 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index 140728ae704..baacbc0cfae 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -10,12 +10,6 @@ public: using FinalizeEvictionFunc = std::function; EvictionCandidates() = default; - EvictionCandidates(EvictionCandidates && other) noexcept - { - candidates = std::move(other.candidates); - candidates_size = std::move(other.candidates_size); - queue_entries_to_invalidate = std::move(other.queue_entries_to_invalidate); - } ~EvictionCandidates(); void add( diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index bf76a307364..12ea2c178bc 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1388,8 +1388,8 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, if (new_settings.max_size != actual_settings.max_size || new_settings.max_elements != actual_settings.max_elements) { - std::optional eviction_candidates; - bool modified_size_limits = false; + EvictionCandidates eviction_candidates; + bool limits_satisfied = false; { cache_is_being_resized.store(true, std::memory_order_relaxed); SCOPE_EXIT({ @@ -1399,15 +1399,12 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, auto cache_lock = lockCache(); FileCacheReserveStat stat; - eviction_candidates.emplace(main_priority->collectCandidatesForEviction( - new_settings.max_size, new_settings.max_elements, 0/* max_candidates_to_evict */, stat, cache_lock)); + limits_satisfied = main_priority->collectCandidatesForEviction( + new_settings.max_size, new_settings.max_elements, 0/* max_candidates_to_evict */, stat, eviction_candidates, cache_lock); - eviction_candidates->removeQueueEntries(cache_lock); + eviction_candidates.removeQueueEntries(cache_lock); - modified_size_limits = main_priority->getSize(cache_lock) <= new_settings.max_size - && main_priority->getElementsCount(cache_lock) <= new_settings.max_elements; - - if (modified_size_limits) + if (limits_satisfied) { main_priority->modifySizeLimits( new_settings.max_size, new_settings.max_elements, new_settings.slru_size_ratio, cache_lock); @@ -1423,16 +1420,16 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, try { - eviction_candidates->evict(); + eviction_candidates.evict(); } catch (...) { auto cache_lock = lockCache(); - eviction_candidates->finalize(nullptr, cache_lock); + eviction_candidates.finalize(nullptr, cache_lock); throw; } - if (modified_size_limits) + if (limits_satisfied) { LOG_INFO(log, "Changed max_size from {} to {}, max_elements from {} to {}", actual_settings.max_size, new_settings.max_size, diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index 042234b9f6c..a727aab68ee 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -147,11 +147,12 @@ public: const CachePriorityGuard::Lock &) = 0; /// Collect eviction `candidates_num` candidates for eviction. - virtual EvictionCandidates collectCandidatesForEviction( + virtual bool collectCandidatesForEviction( size_t desired_size, size_t desired_elements_count, size_t max_candidates_to_evict, FileCacheReserveStat & stat, + EvictionCandidates & candidates, const CachePriorityGuard::Lock &) = 0; virtual bool modifySizeLimits( diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 5d75c9cb18c..e859529f5e7 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -322,14 +322,14 @@ bool LRUFileCachePriority::collectCandidatesForEviction( } } -EvictionCandidates LRUFileCachePriority::collectCandidatesForEviction( +bool LRUFileCachePriority::collectCandidatesForEviction( size_t desired_size, size_t desired_elements_count, size_t max_candidates_to_evict, FileCacheReserveStat & stat, + EvictionCandidates & res, const CachePriorityGuard::Lock & lock) { - EvictionCandidates res; auto stop_condition = [&, this]() { return canFit(0, 0, stat.total_stat.releasable_size, stat.total_stat.releasable_count, @@ -337,7 +337,7 @@ EvictionCandidates LRUFileCachePriority::collectCandidatesForEviction( || (max_candidates_to_evict && res.size() >= max_candidates_to_evict); }; iterateForEviction(res, stat, stop_condition, lock); - return res; + return stop_condition(); } void LRUFileCachePriority::iterateForEviction( diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index 1c8cfa6795f..d31a3fb0f10 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -62,11 +62,12 @@ public: const UserID & user_id, const CachePriorityGuard::Lock &) override; - EvictionCandidates collectCandidatesForEviction( + bool collectCandidatesForEviction( size_t desired_size, size_t desired_elements_count, size_t max_candidates_to_evict, FileCacheReserveStat & stat, + EvictionCandidates & res, const CachePriorityGuard::Lock &) override; void shuffle(const CachePriorityGuard::Lock &) override; diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index 802a88db23f..68bf182dd2e 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -251,42 +251,47 @@ bool SLRUFileCachePriority::collectCandidatesForEvictionInProtected( return true; } -EvictionCandidates SLRUFileCachePriority::collectCandidatesForEviction( +bool SLRUFileCachePriority::collectCandidatesForEviction( size_t desired_size, size_t desired_elements_count, size_t max_candidates_to_evict, FileCacheReserveStat & stat, + EvictionCandidates & res, const CachePriorityGuard::Lock & lock) { const auto desired_probationary_size = getRatio(desired_size, 1 - size_ratio); const auto desired_probationary_elements_num = getRatio(desired_elements_count, 1 - size_ratio); - auto res = probationary_queue.collectCandidatesForEviction( - desired_probationary_size, desired_probationary_elements_num, max_candidates_to_evict, stat, lock); + FileCacheReserveStat probationary_stat; + const bool probationary_limit_satisfied = probationary_queue.collectCandidatesForEviction( + desired_probationary_size, desired_probationary_elements_num, + max_candidates_to_evict, probationary_stat, res, lock); - LOG_TEST(log, "Collected {} eviction candidates from probationary queue (size: {})", - res.size(), stat.total_stat.releasable_size); + stat += probationary_stat; + + LOG_TEST(log, "Collected {} to evict from probationary queue. Total size: {}", + res.size(), probationary_stat.total_stat.releasable_size); chassert(!max_candidates_to_evict || res.size() <= max_candidates_to_evict); chassert(res.size() == stat.total_stat.releasable_count); - if (max_candidates_to_evict && res.size() == max_candidates_to_evict) - return res; + if (max_candidates_to_evict && res.size() >= max_candidates_to_evict) + return probationary_limit_satisfied; const auto desired_protected_size = getRatio(max_size, size_ratio); const auto desired_protected_elements_num = getRatio(max_elements, size_ratio); FileCacheReserveStat protected_stat; - auto res_add = protected_queue.collectCandidatesForEviction( + const bool protected_limit_satisfied = protected_queue.collectCandidatesForEviction( desired_protected_size, desired_protected_elements_num, - max_candidates_to_evict ? max_candidates_to_evict - res.size() : 0, protected_stat, lock); - - LOG_TEST(log, "Collected {} eviction candidates from protected queue (size: {})", - res_add.size(), protected_stat.total_stat.releasable_size); + max_candidates_to_evict - res.size(), protected_stat, res, lock); stat += protected_stat; - res.insert(std::move(res_add), lock); - return res; + + LOG_TEST(log, "Collected {} to evict from protected queue. Total size: {}", + res.size(), protected_stat.total_stat.releasable_size); + + return probationary_limit_satisfied && protected_limit_satisfied; } void SLRUFileCachePriority::downgrade(IteratorPtr iterator, const CachePriorityGuard::Lock & lock) diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h index e837b8541c2..ee3cafe322d 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.h +++ b/src/Interpreters/Cache/SLRUFileCachePriority.h @@ -58,11 +58,12 @@ public: const UserID & user_id, const CachePriorityGuard::Lock &) override; - EvictionCandidates collectCandidatesForEviction( + bool collectCandidatesForEviction( size_t desired_size, size_t desired_elements_count, size_t max_candidates_to_evict, FileCacheReserveStat & stat, + EvictionCandidates & res, const CachePriorityGuard::Lock &) override; void shuffle(const CachePriorityGuard::Lock &) override; From 378d330d9dfa289c413f80c2addaf6dee5503093 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 5 Apr 2024 17:07:43 +0000 Subject: [PATCH 1040/1165] better --- .../0_stateless/03033_parts_splitter_bug_and_index_loading.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.sql b/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.sql index 541ac67fd24..25ec1c8fd80 100644 --- a/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.sql +++ b/tests/queries/0_stateless/03033_parts_splitter_bug_and_index_loading.sql @@ -1,5 +1,7 @@ create table t(a UInt32, b UInt32) engine=MergeTree order by (a, b) settings index_granularity=1; +system stop merges t; + -- for this part the first columns is useless, so we have to use both insert into t select 42, number from numbers_mt(100); From fa01f26defcaeed6c83a0866a9a786250cfdb332 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 5 Apr 2024 17:39:12 +0000 Subject: [PATCH 1041/1165] add test --- .../IO/ReadBufferFromAzureBlobStorage.cpp | 2 +- .../IO/WriteBufferFromAzureBlobStorage.cpp | 2 +- .../isRetryableAzureException.cpp | 10 ++- .../isRetryableAzureException.h | 0 src/Storages/MergeTree/checkDataPart.cpp | 2 +- .../test.py | 4 +- .../test.py | 78 ++++++++++++++++++- .../test_storage_azure_blob_storage/test.py | 8 +- 8 files changed, 90 insertions(+), 16 deletions(-) rename src/{Disks/ObjectStorages => IO}/AzureBlobStorage/isRetryableAzureException.cpp (66%) rename src/{Disks/ObjectStorages => IO}/AzureBlobStorage/isRetryableAzureException.h (100%) diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 68425c5ca18..e4d74b64050 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -3,7 +3,7 @@ #if USE_AZURE_BLOB_STORAGE #include -#include +#include #include #include #include diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index 921f99ffef3..37a189e4d19 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -3,7 +3,7 @@ #if USE_AZURE_BLOB_STORAGE #include -#include +#include #include #include #include diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.cpp b/src/IO/AzureBlobStorage/isRetryableAzureException.cpp similarity index 66% rename from src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.cpp rename to src/IO/AzureBlobStorage/isRetryableAzureException.cpp index e32815e9613..785d8985709 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.cpp +++ b/src/IO/AzureBlobStorage/isRetryableAzureException.cpp @@ -1,10 +1,12 @@ -#include +#include "config.h" + #if USE_AZURE_BLOB_STORAGE +#include namespace DB { -bool isRetryableAzureRequestException(const Azure::Core::RequestFailedException & e) +bool isRetryableAzureException(const Azure::Core::RequestFailedException & e) { /// Always retry transport errors. if (dynamic_cast(&e)) @@ -14,6 +16,6 @@ bool isRetryableAzureRequestException(const Azure::Core::RequestFailedException return e.StatusCode >= Azure::Core::Http::HttpStatusCode::InternalServerError; } -#endif - } + +#endif diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.h b/src/IO/AzureBlobStorage/isRetryableAzureException.h similarity index 100% rename from src/Disks/ObjectStorages/AzureBlobStorage/isRetryableAzureException.h rename to src/IO/AzureBlobStorage/isRetryableAzureException.h diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 208da561118..24826170e55 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py index a7c7b439560..55c2969d8d3 100644 --- a/tests/integration/test_backup_restore_azure_blob_storage/test.py +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -66,11 +66,11 @@ def cluster(): def azure_query( - node, query, expect_error="false", try_num=10, settings={}, query_on_retry=None + node, query, expect_error=False, try_num=10, settings={}, query_on_retry=None ): for i in range(try_num): try: - if expect_error == "true": + if expect_error: return node.query_and_get_error(query, settings=settings) else: return node.query(query, settings=settings) diff --git a/tests/integration/test_merge_tree_azure_blob_storage/test.py b/tests/integration/test_merge_tree_azure_blob_storage/test.py index cffab672bd1..7f77627e793 100644 --- a/tests/integration/test_merge_tree_azure_blob_storage/test.py +++ b/tests/integration/test_merge_tree_azure_blob_storage/test.py @@ -714,7 +714,7 @@ def test_endpoint_error_check(cluster): """ expected_err_msg = "Expected container_name in endpoint" - assert expected_err_msg in azure_query(node, query, expect_error="true") + assert expected_err_msg in azure_query(node, query, expect_error=True) query = f""" DROP TABLE IF EXISTS test SYNC; @@ -731,7 +731,7 @@ def test_endpoint_error_check(cluster): """ expected_err_msg = "Expected account_name in endpoint" - assert expected_err_msg in azure_query(node, query, expect_error="true") + assert expected_err_msg in azure_query(node, query, expect_error=True) query = f""" DROP TABLE IF EXISTS test SYNC; @@ -748,4 +748,76 @@ def test_endpoint_error_check(cluster): """ expected_err_msg = "Expected container_name in endpoint" - assert expected_err_msg in azure_query(node, query, expect_error="true") + assert expected_err_msg in azure_query(node, query, expect_error=True) + + +def get_azure_client(container_name, port): + connection_string = ( + f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + f"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + f"BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;" + ) + + blob_service_client = BlobServiceClient.from_connection_string(connection_string) + return blob_service_client.get_container_client(container_name) + + +def test_azure_broken_parts(cluster): + node = cluster.instances[NODE_NAME] + account_name = "devstoreaccount1" + container_name = "cont5" + port = cluster.azurite_port + + query = f""" + DROP TABLE IF EXISTS t_azure_broken_parts SYNC; + + CREATE TABLE t_azure_broken_parts (a Int32) + ENGINE = MergeTree() ORDER BY tuple() + SETTINGS disk = disk( + type = azure_blob_storage, + endpoint = 'http://azurite1:{port}/{account_name}/{container_name}', + endpoint_contains_account_name = 'true', + account_name = 'devstoreaccount1', + account_key = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', + skip_access_check = 0), min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; + + INSERT INTO t_azure_broken_parts VALUES (1); + """ + + azure_query(node, query) + + result = azure_query(node, "SELECT count() FROM t_azure_broken_parts").strip() + assert int(result) == 1 + + result = azure_query( + node, + "SELECT count() FROM system.detached_parts WHERE table = 't_azure_broken_parts'", + ).strip() + + assert int(result) == 0 + + data_path = azure_query( + node, + "SELECT data_paths[1] FROM system.tables WHERE name = 't_azure_broken_parts'", + ).strip() + + remote_path = azure_query( + node, + f"SELECT remote_path FROM system.remote_data_paths WHERE path || local_path = '{data_path}' || 'all_1_1_0/columns.txt'", + ).strip() + + client = get_azure_client(container_name, port) + client.delete_blob(remote_path) + + azure_query(node, "DETACH TABLE t_azure_broken_parts") + azure_query(node, "ATTACH TABLE t_azure_broken_parts") + + result = azure_query(node, "SELECT count() FROM t_azure_broken_parts").strip() + assert int(result) == 0 + + result = azure_query( + node, + "SELECT count() FROM system.detached_parts WHERE table = 't_azure_broken_parts'", + ).strip() + + assert int(result) == 1 diff --git a/tests/integration/test_storage_azure_blob_storage/test.py b/tests/integration/test_storage_azure_blob_storage/test.py index 7d30265e4f8..aabc9340658 100644 --- a/tests/integration/test_storage_azure_blob_storage/test.py +++ b/tests/integration/test_storage_azure_blob_storage/test.py @@ -36,11 +36,11 @@ def cluster(): def azure_query( - node, query, expect_error="false", try_num=10, settings={}, query_on_retry=None + node, query, expect_error=False, try_num=10, settings={}, query_on_retry=None ): for i in range(try_num): try: - if expect_error == "true": + if expect_error: return node.query_and_get_error(query, settings=settings) else: return node.query(query, settings=settings) @@ -793,7 +793,7 @@ def test_read_from_not_existing_container(cluster): f"'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV', 'auto')" ) expected_err_msg = "container does not exist" - assert expected_err_msg in azure_query(node, query, expect_error="true") + assert expected_err_msg in azure_query(node, query, expect_error=True) def test_function_signatures(cluster): @@ -966,7 +966,7 @@ def test_union_schema_inference_mode(cluster): error = azure_query( node, f"desc azureBlobStorage('{storage_account_url}', 'cont', 'test_union_schema_inference*.jsonl', '{account_name}', '{account_key}', 'auto', 'auto', 'auto') settings schema_inference_mode='union', describe_compact_output=1 format TSV", - expect_error="true", + expect_error=True, ) assert "CANNOT_EXTRACT_TABLE_STRUCTURE" in error From 0bce544779bd881aa3218694545fe5a8017ee9a4 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Fri, 5 Apr 2024 23:07:00 +0200 Subject: [PATCH 1042/1165] Update base/poco/Net/src/HTTPClientSession.cpp Co-authored-by: Nikita Taranov --- base/poco/Net/src/HTTPClientSession.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp index e489ab56b98..c9899266be7 100644 --- a/base/poco/Net/src/HTTPClientSession.cpp +++ b/base/poco/Net/src/HTTPClientSession.cpp @@ -248,7 +248,7 @@ void HTTPClientSession::setKeepAliveRequest(int request) - void HTTPClientSession::setLastRequest(Poco::Timestamp time) +void HTTPClientSession::setLastRequest(Poco::Timestamp time) { if (connected()) { From f766ec678206c0b0e5f0eac0d142583fa47d89cd Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 5 Apr 2024 23:19:30 +0200 Subject: [PATCH 1043/1165] review remarks --- src/Common/HTTPConnectionPool.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index 7f99d6a647f..167aeee68f3 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -213,7 +213,8 @@ public: --total_connections_in_group; - const size_t reduced_warning_limit = limits.warning_limit > 10 ? limits.warning_limit - 20 : 1; + const size_t gap = 20; + const size_t reduced_warning_limit = limits.warning_limit > gap ? limits.warning_limit - gap : 1; if (mute_warning_until > 0 && total_connections_in_group < reduced_warning_limit) { LOG_WARNING(log, "Sessions count is OK in the group {}, count {}", type, total_connections_in_group); @@ -289,8 +290,7 @@ private: auto timeouts = getTimeouts(*this); auto new_connection = lock->getConnection(timeouts); Session::assign(*new_connection); - if (Session::getKeepAliveRequest() == 0) - Session::setKeepAliveRequest(1); + Session::setKeepAliveRequest(Session::getKeepAliveRequest() + 1); } else { @@ -425,7 +425,7 @@ private: ConnectionGroup::Ptr group_, IHTTPConnectionPoolForEndpoint::Metrics metrics_, Args &&... args) - : Session(args...) + : Session(std::forward(args)...) , pool(std::move(pool_)) , group(group_) , metrics(std::move(metrics_)) From c0ffee763cd90da02310e94c99e04d6cc5a2afa6 Mon Sep 17 00:00:00 2001 From: Anita Hammer <166057949+anitahammer@users.noreply.github.com> Date: Sat, 6 Apr 2024 02:48:50 +0300 Subject: [PATCH 1044/1165] Fix contributor name vulnerability --- tests/ci/version_helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 30b0c2d96be..f649732171f 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -357,8 +357,9 @@ def update_contributors( # format: " 1016 Alexey Arno" shortlog = git_runner.run("git shortlog HEAD --summary") + escaping = str.maketrans({"\\": "\\\\", '"': '\\"'}) contributors = sorted( - [c.split(maxsplit=1)[-1].replace('"', r"\"") for c in shortlog.split("\n")], + [c.split(maxsplit=1)[-1].translate(escaping) for c in shortlog.split("\n")], ) contributors = [f' "{c}",' for c in contributors] From aae16fb41e960895c384fcacdbbef4940af18e28 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 6 Apr 2024 11:19:48 +0200 Subject: [PATCH 1045/1165] Remove macro --- src/Common/ThreadStatus.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 3e7f27f4d4f..ad96018a17e 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -141,9 +141,6 @@ void ThreadStatus::initGlobalProfiler([[maybe_unused]] UInt64 global_profiler_re { tryLogCurrentException("ThreadStatus", "Cannot initialize GlobalProfiler"); } -#else - UNUSED(global_profiler_real_time_period); - UNUSED(global_profiler_cpu_time_period); #endif } From 56d6ec5f302f10b02bd7fd1fe47618083626a06c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sat, 6 Apr 2024 12:13:22 +0200 Subject: [PATCH 1046/1165] Update 03023_group_by_use_nulls_analyzer_crashes.sql --- .../0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql index 20cf47e4feb..2f4c8b1c75e 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql @@ -6,6 +6,6 @@ select tuple(array(number)) as x FROM numbers(10) GROUP BY number, array(number) SELECT tuple(number) AS x FROM numbers(10) GROUP BY GROUPING SETS (number) order by x; -SELECT ignore(toFixedString('Lambda as function parameter', 28), toNullable(28), ignore(8)), sum(marks) FROM system.parts GROUP BY GROUPING SETS ((2)) FORMAT Null settings optimize_injective_functions_in_group_by=1, optimize_group_by_function_keys=1, group_by_use_nulls=1; -- { serverError ILLEGAL_AGGREGATION } +SELECT ignore(toFixedString('Lambda as function parameter', 28), toNullable(28), ignore(8)), sum(marks) FROM system.parts WHERE database = currentDatabase() GROUP BY GROUPING SETS ((2)) FORMAT Null settings optimize_injective_functions_in_group_by=1, optimize_group_by_function_keys=1, group_by_use_nulls=1; -- { serverError ILLEGAL_AGGREGATION } SELECT toLowCardinality(materialize('a' AS key)), 'b' AS value GROUP BY key WITH CUBE SETTINGS group_by_use_nulls = 1; From 2573fb62627c76554619c8c046b45eaef02bd13d Mon Sep 17 00:00:00 2001 From: zhongyuankai <872237106@qq.com> Date: Sat, 6 Apr 2024 20:37:37 +0800 Subject: [PATCH 1047/1165] batter --- .../engines/table-engines/special/memory.md | 11 +++++---- src/Storages/MemorySettings.cpp | 15 +++++++++--- .../03032_storage_memory_modify_settings.sql | 23 ++++++++++--------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/docs/en/engines/table-engines/special/memory.md b/docs/en/engines/table-engines/special/memory.md index 4f5d8a1d5e0..f28157ebde2 100644 --- a/docs/en/engines/table-engines/special/memory.md +++ b/docs/en/engines/table-engines/special/memory.md @@ -37,11 +37,6 @@ Upper and lower bounds can be specified to limit Memory engine table size, effec - `max_rows_to_keep` — Maximum rows to keep within memory table where oldest rows are deleted on each insertion (i.e circular buffer). Max rows can exceed the stated limit if the oldest batch of rows to remove falls under the `min_rows_to_keep` limit when adding a large block. - Default value: `0` -**Modify settings** -```sql -ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; -``` - ## Usage {#usage} @@ -50,6 +45,11 @@ ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 100 CREATE TABLE memory (i UInt32) ENGINE = Memory SETTINGS min_rows_to_keep = 100, max_rows_to_keep = 1000; ``` +**Modify settings** +```sql +ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; +``` + **Note:** Both `bytes` and `rows` capping parameters can be set at the same time, however, the lower bounds of `max` and `min` will be adhered to. ## Examples {#examples} @@ -102,3 +102,4 @@ SELECT total_bytes, total_rows FROM system.tables WHERE name = 'memory' and data │ 65536 │ 10000 │ └─────────────┴────────────┘ ``` + diff --git a/src/Storages/MemorySettings.cpp b/src/Storages/MemorySettings.cpp index 4968f2b3b16..7c8c9da209b 100644 --- a/src/Storages/MemorySettings.cpp +++ b/src/Storages/MemorySettings.cpp @@ -44,9 +44,18 @@ ASTPtr MemorySettings::getSettingsChangesQuery() void MemorySettings::sanityCheck() const { - if (min_bytes_to_keep > max_bytes_to_keep - || min_rows_to_keep > max_rows_to_keep) - throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Min. bytes / rows must be set with a max."); + if (min_bytes_to_keep > max_bytes_to_keep) + throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, + "`min_bytes_to_keep` setting cannot be higher than `max_bytes_to_keep`. `min_bytes_to_keep`: {}, `max_bytes_to_keep`: {}", + min_bytes_to_keep, + max_bytes_to_keep); + + + if (min_rows_to_keep > max_rows_to_keep) + throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, + "`min_rows_to_keep` setting cannot be higher than `max_rows_to_keep`. `min_rows_to_keep`: {}, `max_rows_to_keep`: {}", + min_rows_to_keep, + max_rows_to_keep); } } diff --git a/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql index e59a5e4edb6..1507107c37f 100644 --- a/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql +++ b/tests/queries/0_stateless/03032_storage_memory_modify_settings.sql @@ -6,10 +6,10 @@ CREATE TABLE memory (i UInt32) ENGINE = Memory SETTINGS min_bytes_to_keep = 8192 INSERT INTO memory SELECT * FROM numbers(0, 100); -- 1024 bytes INSERT INTO memory SELECT * FROM numbers(0, 3000); -- 16384 bytes -SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 17408 in total ALTER TABLE memory MODIFY SETTING min_bytes_to_keep = 4096, max_bytes_to_keep = 16384; -SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 16384 in total after deleting INSERT INTO memory SELECT * FROM numbers(3000, 10000); -- 65536 bytes SELECT total_bytes FROM system.tables WHERE name = 'memory' and database = currentDatabase(); @@ -33,19 +33,19 @@ DROP TABLE IF EXISTS memory; CREATE TABLE memory (i UInt32) ENGINE = Memory; INSERT INTO memory SELECT * FROM numbers(0, 50); -- 50 rows -SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 50 in total INSERT INTO memory SELECT * FROM numbers(50, 950); -- 950 rows -SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 1000 in total INSERT INTO memory SELECT * FROM numbers(2000, 70); -- 70 rows -SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 1070 in total ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; -SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 1020 in total after deleting INSERT INTO memory SELECT * FROM numbers(3000, 1100); -- 1100 rows -SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 1100 in total after deleting SELECT 'TESTING ADD SETTINGS'; DROP TABLE IF EXISTS memory; @@ -53,16 +53,16 @@ CREATE TABLE memory (i UInt32) ENGINE = Memory; ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; INSERT INTO memory SELECT * FROM numbers(0, 50); -- 50 rows -SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 50 in total INSERT INTO memory SELECT * FROM numbers(50, 950); -- 950 rows -SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 1000 in total INSERT INTO memory SELECT * FROM numbers(2000, 70); -- 70 rows -SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 1020 in total after deleting INSERT INTO memory SELECT * FROM numbers(3000, 1100); -- 1100 rows -SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); +SELECT total_rows FROM system.tables WHERE name = 'memory' and database = currentDatabase(); -- 1100 in total after deleting SELECT 'TESTING INVALID SETTINGS'; DROP TABLE IF EXISTS memory; @@ -73,3 +73,4 @@ ALTER TABLE memory MODIFY SETTING max_rows_to_keep = 1000; ALTER TABLE memory MODIFY SETTING max_bytes_to_keep = 1000; DROP TABLE memory; + From b6dd55827fc5f19a763fa7b06e165cbd8b05fcea Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 6 Apr 2024 14:23:45 +0000 Subject: [PATCH 1048/1165] Fix analyzer with positional arguments in distributed query --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 8 ++++---- src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp | 4 ++++ ..._distributed_query_with_positional_arguments.reference | 3 +++ ...alyzer_distributed_query_with_positional_arguments.sql | 7 +++++++ 4 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/03018_analyzer_distributed_query_with_positional_arguments.reference create mode 100644 tests/queries/0_stateless/03018_analyzer_distributed_query_with_positional_arguments.sql diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 6350266504f..5d710a3fda3 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -6443,7 +6443,7 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR { for (auto & grouping_sets_keys_list_node : query_node_typed.getGroupBy().getNodes()) { - if (settings.enable_positional_arguments) + if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) replaceNodesWithPositionalArguments(grouping_sets_keys_list_node, query_node_typed.getProjection().getNodes(), scope); resolveExpressionNodeList(grouping_sets_keys_list_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); @@ -6465,7 +6465,7 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR } else { - if (settings.enable_positional_arguments) + if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) replaceNodesWithPositionalArguments(query_node_typed.getGroupByNode(), query_node_typed.getProjection().getNodes(), scope); resolveExpressionNodeList(query_node_typed.getGroupByNode(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); @@ -7823,7 +7823,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.hasOrderBy()) { - if (settings.enable_positional_arguments) + if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) replaceNodesWithPositionalArguments(query_node_typed.getOrderByNode(), query_node_typed.getProjection().getNodes(), scope); expandOrderByAll(query_node_typed, settings); @@ -7847,7 +7847,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.hasLimitBy()) { - if (settings.enable_positional_arguments) + if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) replaceNodesWithPositionalArguments(query_node_typed.getLimitByNode(), query_node_typed.getProjection().getNodes(), scope); resolveExpressionNodeList(query_node_typed.getLimitByNode(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index c8d230c87d9..d4545482477 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -68,6 +68,10 @@ std::unique_ptr createLocalPlan( if (context->getSettingsRef().allow_experimental_analyzer) { + /// For Analyzer, identifier in GROUP BY/ORDER BY/LIMIT BY lists has been resolved to + /// ConstantNode in QueryTree if it is an alias of a constant, so we should not replace + /// ConstantNode with ProjectionNode again(https://github.com/ClickHouse/ClickHouse/issues/62289). + new_context->setSetting("enable_positional_arguments", Field(false)); auto interpreter = InterpreterSelectQueryAnalyzer(query_ast, new_context, select_query_options); query_plan = std::make_unique(std::move(interpreter).extractQueryPlan()); } diff --git a/tests/queries/0_stateless/03018_analyzer_distributed_query_with_positional_arguments.reference b/tests/queries/0_stateless/03018_analyzer_distributed_query_with_positional_arguments.reference new file mode 100644 index 00000000000..bb0b1cf658d --- /dev/null +++ b/tests/queries/0_stateless/03018_analyzer_distributed_query_with_positional_arguments.reference @@ -0,0 +1,3 @@ +0 +0 +0 diff --git a/tests/queries/0_stateless/03018_analyzer_distributed_query_with_positional_arguments.sql b/tests/queries/0_stateless/03018_analyzer_distributed_query_with_positional_arguments.sql new file mode 100644 index 00000000000..16ba3b15594 --- /dev/null +++ b/tests/queries/0_stateless/03018_analyzer_distributed_query_with_positional_arguments.sql @@ -0,0 +1,7 @@ +select 0 as x +from remote('127.0.0.{1,2}', system.one) +group by x; + +select 0 as x +from remote('127.0.0.{1,2}', system.one) +order by x; From 933d6e86dab5d0c489134d7c215b4e40f0c19ed0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 6 Apr 2024 19:33:54 +0200 Subject: [PATCH 1049/1165] Remove config --- src/Common/ThreadStatus.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 02bf82e2da3..0c02ab8fdb0 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -1,6 +1,5 @@ #pragma once -#include "config.h" #include #include #include From a6dbaf540882b18950e28ec5f1c9155e34aeadf1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Apr 2024 21:45:26 +0200 Subject: [PATCH 1050/1165] dhparams are not enabled by default --- programs/install/Install.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index 0ff0faff624..20c1a0ad4a8 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -662,7 +662,6 @@ int mainEntryClickHouseInstall(int argc, char ** argv) " \n" " " << (config_dir / "server.crt").string() << "\n" " " << (config_dir / "server.key").string() << "\n" - " " << (config_dir / "dhparam.pem").string() << "\n" " \n" " \n" "\n"; From 664823463b23d00d2aa4293bdea763112b652ddb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Apr 2024 21:46:43 +0200 Subject: [PATCH 1051/1165] Do not create a directory for UDF in clickhouse-client if it does not exist --- .../UserDefined/UserDefinedSQLObjectsDiskStorage.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp index b083c540083..d874612ad04 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp @@ -56,7 +56,6 @@ UserDefinedSQLObjectsDiskStorage::UserDefinedSQLObjectsDiskStorage(const Context , dir_path{makeDirectoryPathCanonical(dir_path_)} , log{getLogger("UserDefinedSQLObjectsLoaderFromDisk")} { - createDirectory(); } @@ -122,7 +121,12 @@ void UserDefinedSQLObjectsDiskStorage::reloadObjects() void UserDefinedSQLObjectsDiskStorage::loadObjectsImpl() { LOG_INFO(log, "Loading user defined objects from {}", dir_path); - createDirectory(); + + if (!std::filesystem::exists(dir_path)) + { + LOG_DEBUG(log, "The directory for user defined objects ({}) does not exist: nothing to load", dir_path); + return; + } std::vector> function_names_and_queries; @@ -157,7 +161,6 @@ void UserDefinedSQLObjectsDiskStorage::loadObjectsImpl() void UserDefinedSQLObjectsDiskStorage::reloadObject(UserDefinedSQLObjectType object_type, const String & object_name) { - createDirectory(); auto ast = tryLoadObject(object_type, object_name); if (ast) setObject(object_name, *ast); @@ -185,6 +188,7 @@ bool UserDefinedSQLObjectsDiskStorage::storeObjectImpl( bool replace_if_exists, const Settings & settings) { + createDirectory(); String file_path = getFilePath(object_type, object_name); LOG_DEBUG(log, "Storing user-defined object {} to file {}", backQuote(object_name), file_path); From 982d3db5b1fa220323c3f7d911b11b375945ca93 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 6 Apr 2024 21:51:28 +0200 Subject: [PATCH 1052/1165] disable window view with analyzer properly --- src/Storages/WindowView/StorageWindowView.cpp | 38 +++++++++++++++++-- src/Storages/WindowView/StorageWindowView.h | 4 ++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 0b822b9aab3..f82f5b07910 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -439,6 +439,7 @@ bool StorageWindowView::optimize( bool cleanup, ContextPtr local_context) { + throwIfWindowViewIsDisabled(local_context); auto storage_ptr = getInnerTable(); auto metadata_snapshot = storage_ptr->getInMemoryMetadataPtr(); return getInnerTable()->optimize(query, metadata_snapshot, partition, final, deduplicate, deduplicate_by_columns, cleanup, local_context); @@ -449,6 +450,7 @@ void StorageWindowView::alter( ContextPtr local_context, AlterLockHolder &) { + throwIfWindowViewIsDisabled(local_context); auto table_id = getStorageID(); StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); @@ -508,8 +510,9 @@ void StorageWindowView::alter( startup(); } -void StorageWindowView::checkAlterIsPossible(const AlterCommands & commands, ContextPtr /*local_context*/) const +void StorageWindowView::checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const { + throwIfWindowViewIsDisabled(local_context); for (const auto & command : commands) { if (!command.isCommentAlter() && command.type != AlterCommand::MODIFY_QUERY) @@ -519,6 +522,7 @@ void StorageWindowView::checkAlterIsPossible(const AlterCommands & commands, Con std::pair StorageWindowView::getNewBlocks(UInt32 watermark) { + throwIfWindowViewIsDisabled(); UInt32 w_start = addTime(watermark, window_kind, -window_num_units, *time_zone); auto inner_table = getInnerTable(); @@ -654,6 +658,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) inline void StorageWindowView::fire(UInt32 watermark) { + throwIfWindowViewIsDisabled(); LOG_TRACE(log, "Watch streams number: {}, target table: {}", watch_streams.size(), target_table_id.empty() ? "None" : target_table_id.getNameForLogs()); @@ -722,6 +727,7 @@ inline void StorageWindowView::fire(UInt32 watermark) ASTPtr StorageWindowView::getSourceTableSelectQuery() { + throwIfWindowViewIsDisabled(); auto query = select_query->clone(); auto & modified_select = query->as(); @@ -947,6 +953,7 @@ UInt32 StorageWindowView::getWindowUpperBound(UInt32 time_sec) void StorageWindowView::addFireSignal(std::set & signals) { + throwIfWindowViewIsDisabled(); std::lock_guard lock(fire_signal_mutex); for (const auto & signal : signals) fire_signal.push_back(signal); @@ -962,6 +969,7 @@ void StorageWindowView::updateMaxTimestamp(UInt32 timestamp) void StorageWindowView::updateMaxWatermark(UInt32 watermark) { + throwIfWindowViewIsDisabled(); if (is_proctime) { max_watermark = watermark; @@ -1014,6 +1022,7 @@ void StorageWindowView::cleanup() void StorageWindowView::threadFuncCleanup() { + throwIfWindowViewIsDisabled(); if (shutdown_called) return; @@ -1033,6 +1042,7 @@ void StorageWindowView::threadFuncCleanup() void StorageWindowView::threadFuncFireProc() { + throwIfWindowViewIsDisabled(); if (shutdown_called) return; @@ -1069,6 +1079,7 @@ void StorageWindowView::threadFuncFireProc() void StorageWindowView::threadFuncFireEvent() { + throwIfWindowViewIsDisabled(); std::lock_guard lock(fire_signal_mutex); LOG_TRACE(log, "Fire events: {}", fire_signal.size()); @@ -1100,6 +1111,7 @@ void StorageWindowView::read( const size_t max_block_size, const size_t num_streams) { + throwIfWindowViewIsDisabled(local_context); if (target_table_id.empty()) return; @@ -1140,6 +1152,7 @@ Pipe StorageWindowView::watch( size_t /*max_block_size*/, const size_t /*num_streams*/) { + throwIfWindowViewIsDisabled(local_context); ASTWatchQuery & query = typeid_cast(*query_info.query); bool has_limit = false; @@ -1178,8 +1191,10 @@ StorageWindowView::StorageWindowView( , clean_interval_usec(context_->getSettingsRef().window_view_clean_interval.totalMicroseconds()) { if (context_->getSettingsRef().allow_experimental_analyzer) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Experimental WINDOW VIEW feature is not supported with new infrastructure for query analysis (the setting 'allow_experimental_analyzer')"); + disabled_due_to_analyzer = true; + + if (mode <= LoadingStrictnessLevel::CREATE) + throwIfWindowViewIsDisabled(); if (!query.select) throw Exception(ErrorCodes::INCORRECT_QUERY, "SELECT query is not specified for {}", getName()); @@ -1243,6 +1258,9 @@ StorageWindowView::StorageWindowView( } } + if (disabled_due_to_analyzer) + return; + clean_cache_task = getContext()->getSchedulePool().createTask(getStorageID().getFullTableName(), [this] { threadFuncCleanup(); }); fire_task = getContext()->getSchedulePool().createTask( getStorageID().getFullTableName(), [this] { is_proctime ? threadFuncFireProc() : threadFuncFireEvent(); }); @@ -1400,6 +1418,7 @@ void StorageWindowView::eventTimeParser(const ASTCreateQuery & query) void StorageWindowView::writeIntoWindowView( StorageWindowView & window_view, const Block & block, ContextPtr local_context) { + window_view.throwIfWindowViewIsDisabled(local_context); while (window_view.modifying_query) std::this_thread::sleep_for(std::chrono::milliseconds(100)); @@ -1589,6 +1608,9 @@ void StorageWindowView::writeIntoWindowView( void StorageWindowView::startup() { + if (disabled_due_to_analyzer) + return; + DatabaseCatalog::instance().addViewDependency(select_table_id, getStorageID()); fire_task->activate(); @@ -1602,6 +1624,8 @@ void StorageWindowView::startup() void StorageWindowView::shutdown(bool) { shutdown_called = true; + if (disabled_due_to_analyzer) + return; fire_condition.notify_all(); @@ -1657,6 +1681,7 @@ Block StorageWindowView::getInputHeader() const const Block & StorageWindowView::getOutputHeader() const { + throwIfWindowViewIsDisabled(); std::lock_guard lock(sample_block_lock); if (!output_header) { @@ -1681,6 +1706,13 @@ StoragePtr StorageWindowView::getTargetTable() const return DatabaseCatalog::instance().getTable(target_table_id, getContext()); } +void StorageWindowView::throwIfWindowViewIsDisabled(ContextPtr local_context) const +{ + if (disabled_due_to_analyzer || (local_context && local_context->getSettingsRef().allow_experimental_analyzer)) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Experimental WINDOW VIEW feature is not supported " + "with new infrastructure for query analysis (the setting 'allow_experimental_analyzer')"); +} + void registerStorageWindowView(StorageFactory & factory) { factory.registerStorage("WindowView", [](const StorageFactory::Arguments & args) diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h index 0b7cd54e3a7..f79867df424 100644 --- a/src/Storages/WindowView/StorageWindowView.h +++ b/src/Storages/WindowView/StorageWindowView.h @@ -271,5 +271,9 @@ private: StoragePtr getSourceTable() const; StoragePtr getInnerTable() const; StoragePtr getTargetTable() const; + + bool disabled_due_to_analyzer = false; + + void throwIfWindowViewIsDisabled(ContextPtr local_context = nullptr) const; }; } From c5e47bbe70e232188e36d0599e29605db4905861 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Apr 2024 21:52:04 +0200 Subject: [PATCH 1053/1165] Add a test --- .../03033_analyzer_query_parameters.sh | 4 ++-- ...udf_user_defined_directory_in_client.reference | 1 + .../03036_udf_user_defined_directory_in_client.sh | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.reference create mode 100755 tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.sh diff --git a/tests/queries/0_stateless/03033_analyzer_query_parameters.sh b/tests/queries/0_stateless/03033_analyzer_query_parameters.sh index c821791e437..cf46067df99 100755 --- a/tests/queries/0_stateless/03033_analyzer_query_parameters.sh +++ b/tests/queries/0_stateless/03033_analyzer_query_parameters.sh @@ -4,5 +4,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -clickhouse-local --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 1" -clickhouse-local --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 0" +${CLICKHOUSE_LOCAL} --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 1" +${CLICKHOUSE_LOCAL} --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 0" diff --git a/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.reference b/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.reference new file mode 100644 index 00000000000..251d054748a --- /dev/null +++ b/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.reference @@ -0,0 +1 @@ +Unknown function diff --git a/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.sh b/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.sh new file mode 100755 index 00000000000..e0a145d8456 --- /dev/null +++ b/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE test (s String) ENGINE = Memory" + +# Calling an unknown function should not lead to creation of a 'user_defined' directory in the current directory +${CLICKHOUSE_CLIENT} --query "INSERT INTO test VALUES (xyz('abc'))" 2>&1 | grep -o -F 'Unknown function' + +ls -ld user_defined 2> /dev/null + +${CLICKHOUSE_CLIENT} --query "DROP TABLE test" From 97281203b8e5009412f58338ff7e7fbab34b431a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 7 Apr 2024 09:51:45 +0000 Subject: [PATCH 1054/1165] Don't access static members through instance, pt. II clang-tidy rightfully complains (-readability-static-accessed-through-instance) that static member functions are accessed through non-static members --- .clang-tidy | 1 - programs/disks/DisksApp.cpp | 2 +- programs/keeper-client/KeeperClient.cpp | 2 +- programs/local/LocalServer.cpp | 2 +- ...regateFunctionExponentialMovingAverage.cpp | 14 +++++----- ...AggregateFunctionKolmogorovSmirnovTest.cpp | 14 +++++----- src/AggregateFunctions/DDSketch.h | 4 +++ src/AggregateFunctions/DDSketch/Store.h | 4 +++ src/Analyzer/Passes/QueryAnalysisPass.cpp | 7 +++-- src/Client/Connection.cpp | 2 +- src/Columns/ColumnObject.cpp | 6 ++--- .../tests/gtest_compressionCodec.cpp | 2 +- src/Daemon/BaseDaemon.cpp | 2 +- src/DataTypes/DataTypeDate32.cpp | 2 +- .../tests/gtest_json_parser.cpp | 4 +-- .../IO/CachedOnDiskReadBufferFromFile.cpp | 2 +- src/Functions/FunctionsConversion.cpp | 4 +-- .../UserDefinedSQLFunctionFactory.cpp | 4 +-- src/IO/MMapReadBufferFromFileWithCache.cpp | 2 +- src/Interpreters/ActionsVisitor.cpp | 4 +-- src/Interpreters/Aggregator.cpp | 6 ++--- src/Interpreters/InterpreterAlterQuery.cpp | 4 +-- .../InterpreterCreateIndexQuery.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 6 ++--- src/Interpreters/Set.cpp | 2 +- src/Interpreters/TreeOptimizer.cpp | 2 +- src/Interpreters/executeQuery.cpp | 2 +- src/Interpreters/tests/gtest_filecache.cpp | 16 ++++++------ src/Loggers/Loggers.cpp | 4 +++ src/Processors/Merges/Algorithms/Graphite.cpp | 26 +++++++++---------- src/Storages/FileLog/StorageFileLog.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/StorageDictionary.cpp | 2 +- src/Storages/StorageFuzzJSON.cpp | 2 +- .../System/StorageSystemFunctions.cpp | 2 +- src/TableFunctions/TableFunctionExplain.cpp | 2 +- 36 files changed, 88 insertions(+), 78 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index c98bee71d1a..dc1cebe9430 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -119,7 +119,6 @@ Checks: [ '-readability-named-parameter', '-readability-redundant-declaration', '-readability-simplify-boolean-expr', - '-readability-static-accessed-through-instance', '-readability-suspicious-call-argument', '-readability-uppercase-literal-suffix', '-readability-use-anyofallof', diff --git a/programs/disks/DisksApp.cpp b/programs/disks/DisksApp.cpp index b7c3c7f5c97..6c768799221 100644 --- a/programs/disks/DisksApp.cpp +++ b/programs/disks/DisksApp.cpp @@ -166,7 +166,7 @@ int DisksApp::main(const std::vector & /*args*/) { String config_path = config().getString("config-file", getDefaultConfigFileName()); ConfigProcessor config_processor(config_path, false, false); - config_processor.setConfigPath(fs::path(config_path).parent_path()); + ConfigProcessor::setConfigPath(fs::path(config_path).parent_path()); auto loaded_config = config_processor.loadConfig(); config().add(loaded_config.configuration.duplicate(), false, false); } diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp index 8297fab5ed9..52d825f30e6 100644 --- a/programs/keeper-client/KeeperClient.cpp +++ b/programs/keeper-client/KeeperClient.cpp @@ -368,7 +368,7 @@ int KeeperClient::main(const std::vector & /* args */) DB::ConfigProcessor config_processor(config().getString("config-file", "config.xml")); /// This will handle a situation when clickhouse is running on the embedded config, but config.d folder is also present. - config_processor.registerEmbeddedConfig("config.xml", ""); + ConfigProcessor::registerEmbeddedConfig("config.xml", ""); auto clickhouse_config = config_processor.loadConfig(); Poco::Util::AbstractConfiguration::Keys keys; diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 05c9830ee2c..72920fbd855 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -122,7 +122,7 @@ void LocalServer::initialize(Poco::Util::Application & self) { const auto config_path = config().getString("config-file", "config.xml"); ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(fs::path(config_path).parent_path()); + ConfigProcessor::setConfigPath(fs::path(config_path).parent_path()); auto loaded_config = config_processor.loadConfig(); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } diff --git a/src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp b/src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp index 8582c8c56fc..3d7d6eff608 100644 --- a/src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp +++ b/src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp @@ -54,30 +54,30 @@ public: { const auto & value = columns[0]->getFloat64(row_num); const auto & time = columns[1]->getFloat64(row_num); - this->data(place).add(value, time, half_decay); + data(place).add(value, time, half_decay); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override { - this->data(place).merge(this->data(rhs), half_decay); + data(place).merge(data(rhs), half_decay); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - writeBinary(this->data(place).value, buf); - writeBinary(this->data(place).time, buf); + writeBinary(data(place).value, buf); + writeBinary(data(place).time, buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override { - readBinary(this->data(place).value, buf); - readBinary(this->data(place).time, buf); + readBinary(data(place).value, buf); + readBinary(data(place).time, buf); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { auto & column = assert_cast &>(to); - column.getData().push_back(this->data(place).get(half_decay)); + column.getData().push_back(data(place).get(half_decay)); } }; diff --git a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp index 882150325be..e1224fae2fb 100644 --- a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp @@ -293,32 +293,32 @@ public: Float64 value = columns[0]->getFloat64(row_num); UInt8 is_second = columns[1]->getUInt(row_num); if (is_second) - this->data(place).addY(value, arena); + data(place).addY(value, arena); else - this->data(place).addX(value, arena); + data(place).addX(value, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - this->data(place).merge(this->data(rhs), arena); + data(place).merge(data(rhs), arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf); + data(place).write(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, arena); + data(place).read(buf, arena); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - if (!this->data(place).size_x || !this->data(place).size_y) + if (!data(place).size_x || !data(place).size_y) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} require both samples to be non empty", getName()); - auto [d_statistic, p_value] = this->data(place).getResult(alternative, method); + auto [d_statistic, p_value] = data(place).getResult(alternative, method); /// Because p-value is a probability. p_value = std::min(1.0, std::max(0.0, p_value)); diff --git a/src/AggregateFunctions/DDSketch.h b/src/AggregateFunctions/DDSketch.h index 65ce7322696..dae935bd43d 100644 --- a/src/AggregateFunctions/DDSketch.h +++ b/src/AggregateFunctions/DDSketch.h @@ -147,6 +147,8 @@ public: negative_store->merge(other.negative_store.get()); } + /// NOLINTBEGIN(readability-static-accessed-through-instance) + void serialize(WriteBuffer& buf) const { // Write the mapping @@ -201,6 +203,8 @@ public: count = static_cast(negative_store->count + zero_count + store->count); } + /// NOLINTEND(readability-static-accessed-through-instance) + private: std::unique_ptr mapping; std::unique_ptr store; diff --git a/src/AggregateFunctions/DDSketch/Store.h b/src/AggregateFunctions/DDSketch/Store.h index 428b2a6c9b8..f12c3875dcf 100644 --- a/src/AggregateFunctions/DDSketch/Store.h +++ b/src/AggregateFunctions/DDSketch/Store.h @@ -87,6 +87,8 @@ public: count += other->count; } + /// NOLINTBEGIN(readability-static-accessed-through-instance) + void serialize(WriteBuffer& buf) const { @@ -179,6 +181,8 @@ public: } } + /// NOLINTEND(readability-static-accessed-through-instance) + private: UInt32 chunk_size; DDSketchEncoding enc; diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index fffb8f7f281..59c02f2b96f 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -5793,7 +5793,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi return result_projection_names; } - FunctionOverloadResolverPtr function = UserDefinedExecutableFunctionFactory::instance().tryGet(function_name, scope.context, parameters); + FunctionOverloadResolverPtr function = UserDefinedExecutableFunctionFactory::instance().tryGet(function_name, scope.context, parameters); /// NOLINT(readability-static-accessed-through-instance) bool is_executable_udf = true; IdentifierResolveScope::ResolvedFunctionsCache * function_cache = nullptr; @@ -5823,7 +5823,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi { std::vector possible_function_names; - auto function_names = UserDefinedExecutableFunctionFactory::instance().getRegisteredNames(scope.context); + auto function_names = UserDefinedExecutableFunctionFactory::instance().getRegisteredNames(scope.context); /// NOLINT(readability-static-accessed-through-instance) possible_function_names.insert(possible_function_names.end(), function_names.begin(), function_names.end()); function_names = UserDefinedSQLFunctionFactory::instance().getAllRegisteredNames(); @@ -5841,8 +5841,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi possible_function_names.push_back(name); } - NamePrompter<2> name_prompter; - auto hints = name_prompter.getHints(function_name, possible_function_names); + auto hints = NamePrompter<2>::getHints(function_name, possible_function_names); throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Function with name '{}' does not exists. In scope {}{}", diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 180942e6b83..f8607bf14c6 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -141,7 +141,7 @@ void Connection::connect(const ConnectionTimeouts & timeouts) async_callback(socket->impl()->sockfd(), connection_timeout, AsyncEventTimeoutType::CONNECT, description, AsyncTaskExecutor::READ | AsyncTaskExecutor::WRITE | AsyncTaskExecutor::ERROR); if (auto err = socket->impl()->socketError()) - socket->impl()->error(err); // Throws an exception + socket->impl()->error(err); // Throws an exception /// NOLINT(readability-static-accessed-through-instance) socket->setBlocking(true); } diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 0a4f90c2262..90ef974010c 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -940,7 +940,7 @@ void ColumnObject::addNestedSubcolumn(const PathInData & key, const FieldInfo & if (nested_node) { /// Find any leaf of Nested subcolumn. - const auto * leaf = subcolumns.findLeaf(nested_node, [&](const auto &) { return true; }); + const auto * leaf = Subcolumns::findLeaf(nested_node, [&](const auto &) { return true; }); assert(leaf); /// Recreate subcolumn with default values and the same sizes of arrays. @@ -983,7 +983,7 @@ const ColumnObject::Subcolumns::Node * ColumnObject::getLeafOfTheSameNested(cons while (current_node) { /// Try to find the first Nested up to the current node. - const auto * node_nested = subcolumns.findParent(current_node, + const auto * node_nested = Subcolumns::findParent(current_node, [](const auto & candidate) { return candidate.isNested(); }); if (!node_nested) @@ -993,7 +993,7 @@ const ColumnObject::Subcolumns::Node * ColumnObject::getLeafOfTheSameNested(cons /// for the last rows. /// If there are no leaves, skip current node and find /// the next node up to the current. - leaf = subcolumns.findLeaf(node_nested, + leaf = Subcolumns::findLeaf(node_nested, [&](const auto & candidate) { return candidate.data.size() > old_size; diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index 16573e035e0..f3f6345a5b5 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -483,7 +483,7 @@ void testTranscoding(Timer & timer, ICompressionCodec & codec, const CodecTestSe ASSERT_TRUE(EqualByteContainers(test_sequence.data_type->getSizeOfValueInMemory(), source_data, decoded)); - const auto header_size = codec.getHeaderSize(); + const auto header_size = ICompressionCodec::getHeaderSize(); const auto compression_ratio = (encoded_size - header_size) / (source_data.size() * 1.0); if (expected_compression_ratio) diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index cc22db3969c..9f10069d452 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -665,7 +665,7 @@ void BaseDaemon::reloadConfiguration() */ config_path = config().getString("config-file", getDefaultConfigFileName()); ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(fs::path(config_path).parent_path()); + ConfigProcessor::setConfigPath(fs::path(config_path).parent_path()); loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true); if (last_configuration != nullptr) diff --git a/src/DataTypes/DataTypeDate32.cpp b/src/DataTypes/DataTypeDate32.cpp index 762552bcb4c..343e498d303 100644 --- a/src/DataTypes/DataTypeDate32.cpp +++ b/src/DataTypes/DataTypeDate32.cpp @@ -18,7 +18,7 @@ SerializationPtr DataTypeDate32::doGetDefaultSerialization() const Field DataTypeDate32::getDefault() const { - return -static_cast(DateLUT::instance().getDayNumOffsetEpoch()); + return -static_cast(DateLUT::instance().getDayNumOffsetEpoch()); /// NOLINT(readability-static-accessed-through-instance) } void registerDataTypeDate32(DataTypeFactory & factory) diff --git a/src/DataTypes/Serializations/tests/gtest_json_parser.cpp b/src/DataTypes/Serializations/tests/gtest_json_parser.cpp index 9b0c8e44d02..1b5b02d579c 100644 --- a/src/DataTypes/Serializations/tests/gtest_json_parser.cpp +++ b/src/DataTypes/Serializations/tests/gtest_json_parser.cpp @@ -34,7 +34,7 @@ TEST(JSONDataParser, ReadJSON) JSONDataParser parser; ReadBufferFromString buf(json_bad); String res; - parser.readJSON(res, buf); + JSONDataParser::readJSON(res, buf); ASSERT_EQ(json1, res); } @@ -44,7 +44,7 @@ TEST(JSONDataParser, ReadJSON) JSONDataParser parser; ReadBufferFromString buf(json_bad); String res; - parser.readJSON(res, buf); + JSONDataParser::readJSON(res, buf); ASSERT_EQ(json2, res); } } diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 1e108b481ee..1fe369832ac 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -346,7 +346,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegment & file_s } auto downloader_id = file_segment.getOrSetDownloader(); - if (downloader_id == file_segment.getCallerId()) + if (downloader_id == FileSegment::getCallerId()) { if (canStartFromCache(file_offset_of_buffer_end, file_segment)) { diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index cc2ec4fb045..5e072d406ad 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -279,7 +279,7 @@ struct ToDate32Transform32Or64Signed static NO_SANITIZE_UNDEFINED Int32 execute(const FromType & from, const DateLUTImpl & time_zone) { - static const Int32 daynum_min_offset = -static_cast(time_zone.getDayNumOffsetEpoch()); + static const Int32 daynum_min_offset = -static_cast(DateLUTImpl::getDayNumOffsetEpoch()); if constexpr (date_time_overflow_behavior == FormatSettings::DateTimeOverflowBehavior::Throw) { @@ -1092,7 +1092,7 @@ struct ConvertThroughParsing { if constexpr (std::is_same_v) { - vec_to[i] = -static_cast(DateLUT::instance().getDayNumOffsetEpoch()); + vec_to[i] = -static_cast(DateLUT::instance().getDayNumOffsetEpoch()); /// NOLINT(readability-static-accessed-through-instance) } else { diff --git a/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp b/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp index e22cd6d0022..e6796874e50 100644 --- a/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp @@ -106,7 +106,7 @@ void UserDefinedSQLFunctionFactory::checkCanBeRegistered(const ContextPtr & cont if (AggregateFunctionFactory::instance().hasNameOrAlias(function_name)) throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS, "The aggregate function '{}' already exists", function_name); - if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context)) + if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context)) /// NOLINT(readability-static-accessed-through-instance) throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS, "User defined executable function '{}' already exists", function_name); validateFunction(assert_cast(create_function_query).function_core, function_name); @@ -118,7 +118,7 @@ void UserDefinedSQLFunctionFactory::checkCanBeUnregistered(const ContextPtr & co AggregateFunctionFactory::instance().hasNameOrAlias(function_name)) throw Exception(ErrorCodes::CANNOT_DROP_FUNCTION, "Cannot drop system function '{}'", function_name); - if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context)) + if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context)) /// NOLINT(readability-static-accessed-through-instance) throw Exception(ErrorCodes::CANNOT_DROP_FUNCTION, "Cannot drop user defined executable function '{}'", function_name); } diff --git a/src/IO/MMapReadBufferFromFileWithCache.cpp b/src/IO/MMapReadBufferFromFileWithCache.cpp index 68c0c7227ca..0cfb60d6527 100644 --- a/src/IO/MMapReadBufferFromFileWithCache.cpp +++ b/src/IO/MMapReadBufferFromFileWithCache.cpp @@ -37,7 +37,7 @@ MMapReadBufferFromFileWithCache::MMapReadBufferFromFileWithCache( MMapReadBufferFromFileWithCache::MMapReadBufferFromFileWithCache( MMappedFileCache & cache, const std::string & file_name, size_t offset) { - mapped = cache.getOrSet(cache.hash(file_name, offset, -1), [&] + mapped = cache.getOrSet(MMappedFileCache::hash(file_name, offset, -1), [&] { return std::make_shared(file_name, offset); }); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 093c266c785..504b7257563 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -1071,7 +1071,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & auto current_context = data.getContext(); - if (UserDefinedExecutableFunctionFactory::instance().has(node.name, current_context)) + if (UserDefinedExecutableFunctionFactory::instance().has(node.name, current_context)) /// NOLINT(readability-static-accessed-through-instance) { Array parameters; if (node.parameters) @@ -1087,7 +1087,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & } } - function_builder = UserDefinedExecutableFunctionFactory::instance().tryGet(node.name, current_context, parameters); + function_builder = UserDefinedExecutableFunctionFactory::instance().tryGet(node.name, current_context, parameters); /// NOLINT(readability-static-accessed-through-instance) } if (!function_builder) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index bd78c7a1bc1..2db07bb77f6 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1056,7 +1056,7 @@ void NO_INLINE Aggregator::executeImplBatch( /// During processing of row #i we will prefetch HashTable cell for row #(i + prefetch_look_ahead). PrefetchingHelper prefetching; - size_t prefetch_look_ahead = prefetching.getInitialLookAheadValue(); + size_t prefetch_look_ahead = PrefetchingHelper::getInitialLookAheadValue(); /// Optimization for special case when there are no aggregate functions. if (params.aggregates_size == 0) @@ -1077,7 +1077,7 @@ void NO_INLINE Aggregator::executeImplBatch( { if constexpr (prefetch && HasPrefetchMemberFunc) { - if (i == row_begin + prefetching.iterationsToMeasure()) + if (i == row_begin + PrefetchingHelper::iterationsToMeasure()) prefetch_look_ahead = prefetching.calcPrefetchLookAhead(); if (i + prefetch_look_ahead < row_end) @@ -1163,7 +1163,7 @@ void NO_INLINE Aggregator::executeImplBatch( if constexpr (prefetch && HasPrefetchMemberFunc) { - if (i == key_start + prefetching.iterationsToMeasure()) + if (i == key_start + PrefetchingHelper::iterationsToMeasure()) prefetch_look_ahead = prefetching.calcPrefetchLookAhead(); if (i + prefetch_look_ahead < row_end) diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 7acaf95becc..2115dc57126 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -56,7 +56,7 @@ InterpreterAlterQuery::InterpreterAlterQuery(const ASTPtr & query_ptr_, ContextP BlockIO InterpreterAlterQuery::execute() { - FunctionNameNormalizer().visit(query_ptr.get()); + FunctionNameNormalizer::visit(query_ptr.get()); const auto & alter = query_ptr->as(); if (alter.alter_object == ASTAlterQuery::AlterObjectType::DATABASE) { @@ -131,7 +131,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) if (modify_query) { // Expand CTE before filling default database - ApplyWithSubqueryVisitor().visit(*modify_query); + ApplyWithSubqueryVisitor::visit(*modify_query); } /// Add default database to table identifiers that we can encounter in e.g. default expressions, mutation expression, etc. diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index 6045b5d2e24..a439cb672c8 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -25,7 +25,7 @@ namespace ErrorCodes BlockIO InterpreterCreateIndexQuery::execute() { - FunctionNameNormalizer().visit(query_ptr.get()); + FunctionNameNormalizer::visit(query_ptr.get()); auto current_context = getContext(); const auto & create_index = query_ptr->as(); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 7c3bed7388c..c0da510a68b 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1114,7 +1114,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) // Table SQL definition is available even if the table is detached (even permanently) auto query = database->getCreateTableQuery(create.getTable(), getContext()); - FunctionNameNormalizer().visit(query.get()); + FunctionNameNormalizer::visit(query.get()); auto create_query = query->as(); if (!create.is_dictionary && create_query.is_dictionary) @@ -1184,7 +1184,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.select && create.isView()) { // Expand CTE before filling default database - ApplyWithSubqueryVisitor().visit(*create.select); + ApplyWithSubqueryVisitor::visit(*create.select); AddDefaultDatabaseVisitor visitor(getContext(), current_database); visitor.visit(*create.select); } @@ -1763,7 +1763,7 @@ BlockIO InterpreterCreateQuery::executeQueryOnCluster(ASTCreateQuery & create) BlockIO InterpreterCreateQuery::execute() { - FunctionNameNormalizer().visit(query_ptr.get()); + FunctionNameNormalizer::visit(query_ptr.get()); auto & create = query_ptr->as(); bool is_create_database = create.database && !create.table; diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 8f11754b3be..d1520c92dbc 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -168,7 +168,7 @@ void Set::setHeader(const ColumnsWithTypeAndName & header) } /// Choose data structure to use for the set. - data.init(data.chooseMethod(key_columns, key_sizes)); + data.init(SetVariants::chooseMethod(key_columns, key_sizes)); } void Set::fillSetElements() diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index a341dae32fa..7b979088170 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -144,7 +144,7 @@ void optimizeGroupBy(ASTSelectQuery * select_query, ContextPtr context) } else { - FunctionOverloadResolverPtr function_builder = UserDefinedExecutableFunctionFactory::instance().tryGet(function->name, context); + FunctionOverloadResolverPtr function_builder = UserDefinedExecutableFunctionFactory::instance().tryGet(function->name, context); /// NOLINT(readability-static-accessed-through-instance) if (!function_builder) function_builder = function_factory.get(function->name, context); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index ea2f69bd2b1..c11ddc3467a 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -923,7 +923,7 @@ static std::tuple executeQueryImpl( /// Propagate WITH statement to children ASTSelect. if (settings.enable_global_with_statement) { - ApplyWithGlobalVisitor().visit(ast); + ApplyWithGlobalVisitor::visit(ast); } { diff --git a/src/Interpreters/tests/gtest_filecache.cpp b/src/Interpreters/tests/gtest_filecache.cpp index 09afb01d7ff..41191ba1605 100644 --- a/src/Interpreters/tests/gtest_filecache.cpp +++ b/src/Interpreters/tests/gtest_filecache.cpp @@ -364,7 +364,7 @@ TEST_F(FileCacheTest, LRUPolicy) std::cerr << "Step 1\n"; auto cache = DB::FileCache("1", settings); cache.initialize(); - auto key = cache.createKeyForPath("key1"); + auto key = DB::FileCache::createKeyForPath("key1"); auto get_or_set = [&](size_t offset, size_t size) { @@ -728,7 +728,7 @@ TEST_F(FileCacheTest, LRUPolicy) auto cache2 = DB::FileCache("2", settings); cache2.initialize(); - auto key = cache2.createKeyForPath("key1"); + auto key = DB::FileCache::createKeyForPath("key1"); /// Get [2, 29] assertEqual( @@ -747,7 +747,7 @@ TEST_F(FileCacheTest, LRUPolicy) fs::create_directories(settings2.base_path); auto cache2 = DB::FileCache("3", settings2); cache2.initialize(); - auto key = cache2.createKeyForPath("key1"); + auto key = DB::FileCache::createKeyForPath("key1"); /// Get [0, 24] assertEqual( @@ -762,7 +762,7 @@ TEST_F(FileCacheTest, LRUPolicy) auto cache = FileCache("4", settings); cache.initialize(); - const auto key = cache.createKeyForPath("key10"); + const auto key = FileCache::createKeyForPath("key10"); const auto key_path = cache.getKeyPath(key, user); cache.removeAllReleasable(user.user_id); @@ -786,7 +786,7 @@ TEST_F(FileCacheTest, LRUPolicy) auto cache = DB::FileCache("5", settings); cache.initialize(); - const auto key = cache.createKeyForPath("key10"); + const auto key = FileCache::createKeyForPath("key10"); const auto key_path = cache.getKeyPath(key, user); cache.removeAllReleasable(user.user_id); @@ -823,7 +823,7 @@ TEST_F(FileCacheTest, writeBuffer) segment_settings.kind = FileSegmentKind::Temporary; segment_settings.unbounded = true; - auto cache_key = cache.createKeyForPath(key); + auto cache_key = FileCache::createKeyForPath(key); auto holder = cache.set(cache_key, 0, 3, segment_settings, user); /// The same is done in TemporaryDataOnDisk::createStreamToCacheFile. std::filesystem::create_directories(cache.getKeyPath(cache_key, user)); @@ -949,7 +949,7 @@ TEST_F(FileCacheTest, temporaryData) const auto user = FileCache::getCommonUser(); auto tmp_data_scope = std::make_shared(nullptr, &file_cache, TemporaryDataOnDiskSettings{}); - auto some_data_holder = file_cache.getOrSet(file_cache.createKeyForPath("some_data"), 0, 5_KiB, 5_KiB, CreateFileSegmentSettings{}, 0, user); + auto some_data_holder = file_cache.getOrSet(FileCache::createKeyForPath("some_data"), 0, 5_KiB, 5_KiB, CreateFileSegmentSettings{}, 0, user); { ASSERT_EQ(some_data_holder->size(), 5); @@ -1199,7 +1199,7 @@ TEST_F(FileCacheTest, SLRUPolicy) { auto cache = DB::FileCache(std::to_string(++file_cache_name), settings); cache.initialize(); - auto key = cache.createKeyForPath("key1"); + auto key = FileCache::createKeyForPath("key1"); auto add_range = [&](size_t offset, size_t size) { diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index c5862b82f34..f794ad336e2 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -53,6 +53,8 @@ static std::string renderFileNameTemplate(time_t now, const std::string & file_p return path.replace_filename(ss.str()); } +/// NOLINTBEGIN(readability-static-accessed-through-instance) + void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger /*_root*/, const std::string & cmd_name) { auto current_logger = config.getString("logger", ""); @@ -393,6 +395,8 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log } } +/// NOLINTEND(readability-static-accessed-through-instance) + void Loggers::closeLogs(Poco::Logger & logger) { if (log_file) diff --git a/src/Processors/Merges/Algorithms/Graphite.cpp b/src/Processors/Merges/Algorithms/Graphite.cpp index d175ec8d44b..0865ec8c0bb 100644 --- a/src/Processors/Merges/Algorithms/Graphite.cpp +++ b/src/Processors/Merges/Algorithms/Graphite.cpp @@ -86,7 +86,7 @@ static const Graphite::Pattern undef_pattern = .regexp_str = "", .function = nullptr, .retentions = Graphite::Retentions(), - .type = undef_pattern.TypeUndef, + .type = Graphite::Pattern::TypeUndef, }; inline static const Patterns & selectPatternsForMetricType(const Graphite::Params & params, std::string_view path) @@ -118,18 +118,18 @@ Graphite::RollupRule selectPatternForPath( if (!pattern.regexp) { /// Default pattern - if (first_match->type == first_match->TypeUndef && pattern.type == pattern.TypeAll) + if (first_match->type == Graphite::Pattern::TypeUndef && pattern.type == Graphite::Pattern::TypeAll) { /// There is only default pattern for both retention and aggregation return {&pattern, &pattern}; } if (pattern.type != first_match->type) { - if (first_match->type == first_match->TypeRetention) + if (first_match->type == Graphite::Pattern::TypeRetention) { return {first_match, &pattern}; } - if (first_match->type == first_match->TypeAggregation) + if (first_match->type == Graphite::Pattern::TypeAggregation) { return {&pattern, first_match}; } @@ -140,23 +140,23 @@ Graphite::RollupRule selectPatternForPath( if (pattern.regexp->match(path.data(), path.size())) { /// General pattern with matched path - if (pattern.type == pattern.TypeAll) + if (pattern.type == Graphite::Pattern::TypeAll) { /// Only for not default patterns with both function and retention parameters return {&pattern, &pattern}; } - if (first_match->type == first_match->TypeUndef) + if (first_match->type == Graphite::Pattern::TypeUndef) { first_match = &pattern; continue; } if (pattern.type != first_match->type) { - if (first_match->type == first_match->TypeRetention) + if (first_match->type == Graphite::Pattern::TypeRetention) { return {first_match, &pattern}; } - if (first_match->type == first_match->TypeAggregation) + if (first_match->type == Graphite::Pattern::TypeAggregation) { return {&pattern, first_match}; } @@ -415,24 +415,24 @@ static const Pattern & appendGraphitePattern( if (!pattern.function) { - pattern.type = pattern.TypeRetention; + pattern.type = Graphite::Pattern::TypeRetention; } else if (pattern.retentions.empty()) { - pattern.type = pattern.TypeAggregation; + pattern.type = Graphite::Pattern::TypeAggregation; } else { - pattern.type = pattern.TypeAll; + pattern.type = Graphite::Pattern::TypeAll; } - if (pattern.type & pattern.TypeAggregation) /// TypeAggregation or TypeAll + if (pattern.type & Graphite::Pattern::TypeAggregation) /// TypeAggregation or TypeAll if (pattern.function->allocatesMemoryInArena()) throw Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Aggregate function {} isn't supported in GraphiteMergeTree", pattern.function->getName()); /// retention should be in descending order of age. - if (pattern.type & pattern.TypeRetention) /// TypeRetention or TypeAll + if (pattern.type & Graphite::Pattern::TypeRetention) /// TypeRetention or TypeAll ::sort(pattern.retentions.begin(), pattern.retentions.end(), compareRetentions); patterns.emplace_back(pattern); diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 7b0cfdf6a6c..f24b2d4f278 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -467,7 +467,7 @@ void StorageFileLog::openFilesAndSetPos() auto & reader = file_ctx.reader.value(); assertStreamGood(reader); - reader.seekg(0, reader.end); + reader.seekg(0, reader.end); /// NOLINT(readability-static-accessed-through-instance) assertStreamGood(reader); auto file_end = reader.tellg(); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 461d9a31eaa..a301c229c77 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2994,7 +2994,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context commands.apply(new_metadata, local_context); - if (commands.hasInvertedIndex(new_metadata) && !settings.allow_experimental_inverted_index) + if (AlterCommands::hasInvertedIndex(new_metadata) && !settings.allow_experimental_inverted_index) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental Inverted Index feature is not enabled (turn on setting 'allow_experimental_inverted_index')"); diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index a0c4156a704..447fd87cdc9 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -116,7 +116,7 @@ StorageDictionary::StorageDictionary( : StorageDictionary( table_id, table_id.getFullNameNotQuoted(), - context_->getExternalDictionariesLoader().getDictionaryStructure(*dictionary_configuration), + context_->getExternalDictionariesLoader().getDictionaryStructure(*dictionary_configuration), /// NOLINT(readability-static-accessed-through-instance) dictionary_configuration->getString("dictionary.comment", ""), Location::SameDatabaseAndNameAsDictionary, context_) diff --git a/src/Storages/StorageFuzzJSON.cpp b/src/Storages/StorageFuzzJSON.cpp index 918f54b1672..fbfc67f4c7c 100644 --- a/src/Storages/StorageFuzzJSON.cpp +++ b/src/Storages/StorageFuzzJSON.cpp @@ -437,7 +437,7 @@ void fuzzJSONObject( bool first = true; for (const auto & ptr : node_list) { - if (node_count >= config.value_number_limit) + if (node_count >= StorageFuzzJSON::Configuration::value_number_limit) break; WriteBufferFromOwnString child_out; diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index 967132e4d4a..c5c91294801 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -179,7 +179,7 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c } const auto & user_defined_executable_functions_factory = UserDefinedExecutableFunctionFactory::instance(); - const auto & user_defined_executable_functions_names = user_defined_executable_functions_factory.getRegisteredNames(context); + const auto & user_defined_executable_functions_names = user_defined_executable_functions_factory.getRegisteredNames(context); /// NOLINT(readability-static-accessed-through-instance) for (const auto & function_name : user_defined_executable_functions_names) { fillRow(res_columns, function_name, 0, {0}, "", FunctionOrigin::EXECUTABLE_USER_DEFINED, user_defined_executable_functions_factory); diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp index 8607597fa67..df2835dd630 100644 --- a/src/TableFunctions/TableFunctionExplain.cpp +++ b/src/TableFunctions/TableFunctionExplain.cpp @@ -136,7 +136,7 @@ void TableFunctionExplain::parseArguments(const ASTPtr & ast_function, ContextPt ColumnsDescription TableFunctionExplain::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const { - Block sample_block = getInterpreter(context).getSampleBlock(query->as()->getKind()); + Block sample_block = getInterpreter(context).getSampleBlock(query->as()->getKind()); /// NOLINT(readability-static-accessed-through-instance) ColumnsDescription columns_description; for (const auto & column : sample_block.getColumnsWithTypeAndName()) columns_description.add(ColumnDescription(column.name, column.type)); From 12813be9b859f21f9b2eb6424b1cdc3d95b32949 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 3 Apr 2024 13:08:52 +0000 Subject: [PATCH 1055/1165] Ignore queries involving system tables by query cache Fixes #59071 --- docs/en/operations/query-cache.md | 7 ++- docs/en/operations/settings/settings.md | 12 +++++ src/Common/ErrorCodes.cpp | 1 + src/Core/Settings.h | 1 + src/Core/SettingsEnums.cpp | 4 ++ src/Core/SettingsEnums.h | 9 ++++ src/Interpreters/Cache/QueryCache.cpp | 42 ++++++++++++++++ src/Interpreters/Cache/QueryCache.h | 3 ++ src/Interpreters/executeQuery.cpp | 14 +++++- .../02494_query_cache_system_tables.reference | 13 +++++ .../02494_query_cache_system_tables.sql | 48 +++++++++++++++++++ 11 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02494_query_cache_system_tables.reference create mode 100644 tests/queries/0_stateless/02494_query_cache_system_tables.sql diff --git a/docs/en/operations/query-cache.md b/docs/en/operations/query-cache.md index a8532bc22b7..7a920671fc2 100644 --- a/docs/en/operations/query-cache.md +++ b/docs/en/operations/query-cache.md @@ -67,8 +67,7 @@ SETTINGS use_query_cache = true, enable_writes_to_query_cache = false; For maximum control, it is generally recommended to provide settings `use_query_cache`, `enable_writes_to_query_cache` and `enable_reads_from_query_cache` only with specific queries. It is also possible to enable caching at user or profile level (e.g. via `SET -use_query_cache = true`) but one should keep in mind that all `SELECT` queries including monitoring or debugging queries to system tables -may return cached results then. +use_query_cache = true`) but one should keep in mind that all `SELECT` queries may return cached results then. The query cache can be cleared using statement `SYSTEM DROP QUERY CACHE`. The content of the query cache is displayed in system table [system.query_cache](system-tables/query_cache.md). The number of query cache hits and misses since database start are shown as events @@ -175,6 +174,10 @@ Also, results of queries with non-deterministic functions are not cached by defa To force caching of results of queries with non-deterministic functions regardless, use setting [query_cache_nondeterministic_function_handling](settings/settings.md#query-cache-nondeterministic-function-handling). +Results of queries that involve system tables, e.g. `system.processes` or `information_schema.tables`, are not cached by default. To force +caching of results of queries with system tables regardless, use setting +[query_cache_system_table_handling](settings/settings.md#query-cache-system-table-handling). + :::note Prior to ClickHouse v23.11, setting 'query_cache_store_results_of_queries_with_nondeterministic_functions = 0 / 1' controlled whether results of queries with non-deterministic results were cached. In newer ClickHouse versions, this setting is obsolete and has no effect. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index e4e7be83f7d..a95c83fad3a 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1689,6 +1689,18 @@ Possible values: Default value: `throw`. +## query_cache_system_table_handling {#query-cache-system-table-handling} + +Controls how the [query cache](../query-cache.md) handles `SELECT` queries against system tables, i.e. tables in databases `system.*` and `information_schema.*`. + +Possible values: + +- `'throw'` - Throw an exception and don't cache the query result. +- `'save'` - Cache the query result. +- `'ignore'` - Don't cache the query result and don't throw an exception. + +Default value: `throw`. + ## query_cache_min_query_runs {#query-cache-min-query-runs} Minimum number of times a `SELECT` query must run before its result is stored in the [query cache](../query-cache.md). diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index af609fabb8f..97a339b2bac 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -597,6 +597,7 @@ M(716, CANNOT_FORGET_PARTITION) \ M(717, EXPERIMENTAL_FEATURE_ERROR) \ M(718, TOO_SLOW_PARSING) \ + M(719, QUERY_CACHE_USED_WITH_SYSTEM_TABLE) \ \ M(900, DISTRIBUTED_CACHE_ERROR) \ M(901, CANNOT_USE_DISTRIBUTED_CACHE) \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d5ea9534e6c..78e482c6090 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -669,6 +669,7 @@ class IColumn; M(Bool, enable_writes_to_query_cache, true, "Enable storing results of SELECT queries in the query cache", 0) \ M(Bool, enable_reads_from_query_cache, true, "Enable reading results of SELECT queries from the query cache", 0) \ M(QueryCacheNondeterministicFunctionHandling, query_cache_nondeterministic_function_handling, QueryCacheNondeterministicFunctionHandling::Throw, "How the query cache handles queries with non-deterministic functions, e.g. now()", 0) \ + M(QueryCacheSystemTableHandling, query_cache_system_table_handling, QueryCacheSystemTableHandling::Throw, "How the query cache handles queries against system tables, i.e. tables in databases 'system.*' and 'information_schema.*'", 0) \ M(UInt64, query_cache_max_size_in_bytes, 0, "The maximum amount of memory (in bytes) the current user may allocate in the query cache. 0 means unlimited. ", 0) \ M(UInt64, query_cache_max_entries, 0, "The maximum number of query results the current user may store in the query cache. 0 means unlimited.", 0) \ M(UInt64, query_cache_min_query_runs, 0, "Minimum number a SELECT query must run before its result is stored in the query cache", 0) \ diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index e47174a3b9d..0caf6e8d609 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -87,6 +87,10 @@ IMPLEMENT_SETTING_ENUM(QueryCacheNondeterministicFunctionHandling, ErrorCodes::B {"save", QueryCacheNondeterministicFunctionHandling::Save}, {"ignore", QueryCacheNondeterministicFunctionHandling::Ignore}}) +IMPLEMENT_SETTING_ENUM(QueryCacheSystemTableHandling, ErrorCodes::BAD_ARGUMENTS, + {{"throw", QueryCacheSystemTableHandling::Throw}, + {"save", QueryCacheSystemTableHandling::Save}, + {"ignore", QueryCacheSystemTableHandling::Ignore}}) IMPLEMENT_SETTING_ENUM(DateTimeInputFormat, ErrorCodes::BAD_ARGUMENTS, {{"basic", FormatSettings::DateTimeInputFormat::Basic}, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 0aa8216bb85..b17ff11d428 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -184,6 +184,15 @@ enum class QueryCacheNondeterministicFunctionHandling DECLARE_SETTING_ENUM(QueryCacheNondeterministicFunctionHandling) +/// How the query cache handles queries against system tables, tables in databases 'system.*' and 'information_schema.*' +enum class QueryCacheSystemTableHandling +{ + Throw, + Save, + Ignore +}; + +DECLARE_SETTING_ENUM(QueryCacheSystemTableHandling) DECLARE_SETTING_ENUM_WITH_RENAME(DateTimeInputFormat, FormatSettings::DateTimeInputFormat) diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index 151f2ea06cc..8a76378f14c 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -2,8 +2,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -52,7 +54,40 @@ struct HasNonDeterministicFunctionsMatcher } }; +struct HasSystemTablesMatcher +{ + struct Data + { + const ContextPtr context; + bool has_system_tables = false; + }; + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } + + static void visit(const ASTPtr & node, Data & data) + { + if (data.has_system_tables) + return; + + if (const auto * identifier = node->as()) + { + StorageID storage_id = identifier->getTableId(); + if (!storage_id.hasDatabase()) + /// The common case that a database name was not explicitly specified in the SQL. However, isPredefinedTable() is AST-based + /// and assumes that a database name was specified. This bites us in this edge situation: + /// USE SYSTEM; + /// SELECT * FROM PROCESSES; -- instead of SYSTEM.PROCESSES + /// In this case, don't call isPredefinedTable() (to avoid exceptions) and accept that the behavior is not 100% kosher. + return; + bool is_predefined_table = DatabaseCatalog::instance().isPredefinedTable(storage_id); + if (is_predefined_table) + data.has_system_tables = true; + } + } +}; + using HasNonDeterministicFunctionsVisitor = InDepthNodeVisitor; +using HasSystemTablesVisitor = InDepthNodeVisitor; } @@ -63,6 +98,13 @@ bool astContainsNonDeterministicFunctions(ASTPtr ast, ContextPtr context) return finder_data.has_non_deterministic_functions; } +bool astContainsSystemTables(ASTPtr ast, ContextPtr context) +{ + HasSystemTablesMatcher::Data finder_data{context}; + HasSystemTablesVisitor(finder_data).visit(ast); + return finder_data.has_system_tables; +} + namespace { diff --git a/src/Interpreters/Cache/QueryCache.h b/src/Interpreters/Cache/QueryCache.h index c574f3085e3..a06f504de65 100644 --- a/src/Interpreters/Cache/QueryCache.h +++ b/src/Interpreters/Cache/QueryCache.h @@ -17,6 +17,9 @@ namespace DB /// Does AST contain non-deterministic functions like rand() and now()? bool astContainsNonDeterministicFunctions(ASTPtr ast, ContextPtr context); +/// Does AST contain system tables like "system.processes"? +bool astContainsSystemTables(ASTPtr ast, ContextPtr context); + /// Maps queries to query results. Useful to avoid repeated query calculation. /// /// The cache does not aim to be transactionally consistent (which is difficult to get right). For example, the cache is not invalidated diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index ea2f69bd2b1..02fced5850b 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -97,6 +97,7 @@ namespace DB namespace ErrorCodes { extern const int QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS; + extern const int QUERY_CACHE_USED_WITH_SYSTEM_TABLE; extern const int INTO_OUTFILE_NOT_ALLOWED; extern const int INVALID_TRANSACTION; extern const int LOGICAL_ERROR; @@ -1198,15 +1199,26 @@ static std::tuple executeQueryImpl( /// top of the pipeline which stores the result in the query cache. if (can_use_query_cache && settings.enable_writes_to_query_cache) { + /// Only use the query cache if the query does not contain non-deterministic functions or system tables (which are typically non-deterministic) + const bool ast_contains_nondeterministic_functions = astContainsNonDeterministicFunctions(ast, context); + const bool ast_contains_system_tables = astContainsSystemTables(ast, context); + const QueryCacheNondeterministicFunctionHandling nondeterministic_function_handling = settings.query_cache_nondeterministic_function_handling; + const QueryCacheSystemTableHandling system_table_handling = settings.query_cache_system_table_handling; if (ast_contains_nondeterministic_functions && nondeterministic_function_handling == QueryCacheNondeterministicFunctionHandling::Throw) throw Exception(ErrorCodes::QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS, "The query result was not cached because the query contains a non-deterministic function." " Use setting `query_cache_nondeterministic_function_handling = 'save'` or `= 'ignore'` to cache the query result regardless or to omit caching"); - if (!ast_contains_nondeterministic_functions || nondeterministic_function_handling == QueryCacheNondeterministicFunctionHandling::Save) + if (ast_contains_system_tables && system_table_handling == QueryCacheSystemTableHandling::Throw) + throw Exception(ErrorCodes::QUERY_CACHE_USED_WITH_SYSTEM_TABLE, + "The query result was not cached because the query contains a system table." + " Use setting `query_cache_system_table_handling = 'save'` or `= 'ignore'` to cache the query result regardless or to omit caching"); + + if ((!ast_contains_nondeterministic_functions || nondeterministic_function_handling == QueryCacheNondeterministicFunctionHandling::Save) + && (!ast_contains_system_tables || system_table_handling == QueryCacheSystemTableHandling::Save)) { QueryCache::Key key( ast, res.pipeline.getHeader(), diff --git a/tests/queries/0_stateless/02494_query_cache_system_tables.reference b/tests/queries/0_stateless/02494_query_cache_system_tables.reference new file mode 100644 index 00000000000..e41e365766e --- /dev/null +++ b/tests/queries/0_stateless/02494_query_cache_system_tables.reference @@ -0,0 +1,13 @@ +The Default for query_cache_system_table_handling is = throw +0 +Check behavior of query_cache_system_table_handling = throw +0 +Check behavior of query_cache_system_table_handling = save +0 +1 +Check behavior of query_cache_system_table_handling = ignore +0 +0 +Other tests +0 +0 diff --git a/tests/queries/0_stateless/02494_query_cache_system_tables.sql b/tests/queries/0_stateless/02494_query_cache_system_tables.sql new file mode 100644 index 00000000000..89ea9d2b6b7 --- /dev/null +++ b/tests/queries/0_stateless/02494_query_cache_system_tables.sql @@ -0,0 +1,48 @@ +-- Tags: no-parallel +-- Tag no-parallel: Messes with internal cache + +SYSTEM DROP QUERY CACHE; + +SELECT 'The Default for query_cache_system_table_handling is = throw'; +-- Test that the query cache rejects queries that involve system tables. +SELECT * FROM system.one SETTINGS use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } +SELECT count(*) FROM system.query_cache; + +SYSTEM DROP QUERY CACHE; + +SELECT 'Check behavior of query_cache_system_table_handling = throw'; +-- Test that the query cache rejects queries that involve system tables. +SELECT * FROM system.one SETTINGS use_query_cache = 1, query_cache_system_table_handling = 'throw'; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } +SELECT count(*) FROM system.query_cache; + +SYSTEM DROP QUERY CACHE; + +SELECT 'Check behavior of query_cache_system_table_handling = save'; +-- Test that the query cache saves the result of queries that involve system tables. +SELECT * FROM system.one SETTINGS use_query_cache = 1, query_cache_system_table_handling = 'save'; +SELECT count(*) FROM system.query_cache; + +SYSTEM DROP QUERY CACHE; + +SELECT 'Check behavior of query_cache_system_table_handling = ignore'; +-- Test that the query cache ignores the result of queries that involve system tables. +SELECT * FROM system.one SETTINGS use_query_cache = 1, query_cache_system_table_handling = 'ignore'; +SELECT count(*) FROM system.query_cache; + +SYSTEM DROP QUERY CACHE; + +SELECT 'Other tests'; + +-- Edge case which doesn't work well due to conceptual reasons (QueryCache is AST-based), test it anyways to have it documented. +USE system; +SELECT * FROM one SETTINGS use_query_cache = 1; -- doesn't throw but should + +-- This query uses system.zero internally. Since the query cache works at AST level it does not "see' system.zero and must not complain. +SELECT * SETTINGS use_query_cache = 1; + +-- information_schema is also treated as a system table +SELECT * FROM information_schema.tables SETTINGS use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } +SELECT * FROM INFORMATION_SCHEMA.TABLES SETTINGS use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } + +-- Cleanup +SYSTEM DROP QUERY CACHE; From a4bec47a44ebb257f298f0254be0c2a7c3f62a95 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 7 Apr 2024 14:05:08 +0200 Subject: [PATCH 1056/1165] fix tests --- tests/queries/0_stateless/01056_window_view_proc_hop_watch.py | 2 ++ .../01059_window_view_event_hop_watch_strict_asc.py | 2 ++ .../0_stateless/01062_window_view_event_hop_watch_asc.py | 2 ++ .../0_stateless/01065_window_view_event_hop_watch_bounded.py | 2 ++ tests/queries/0_stateless/01070_window_view_watch_events.py | 2 ++ .../0_stateless/01078_window_view_alter_query_watch.py | 4 ++++ tests/queries/0_stateless/01082_window_view_watch_limit.py | 2 ++ 7 files changed, 16 insertions(+) diff --git a/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py b/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py index 4c3e3ead455..2db14fcdddf 100755 --- a/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py +++ b/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py @@ -28,6 +28,8 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) + client2.send("SET allow_experimental_analyzer = 0") + client2.expect(prompt) client1.send("CREATE DATABASE IF NOT EXISTS 01056_window_view_proc_hop_watch") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py index 9adff06442e..2323ee5c838 100755 --- a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py +++ b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py @@ -26,6 +26,8 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client1.send("SET window_view_heartbeat_interval = 1") client1.expect(prompt) + client2.send("SET allow_experimental_analyzer = 0") + client2.expect(prompt) client1.send("CREATE DATABASE IF NOT EXISTS db_01059_event_hop_watch_strict_asc") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py index bb40b1df2f0..db9e8cef6c5 100755 --- a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py +++ b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py @@ -28,6 +28,8 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) + client2.send("SET allow_experimental_analyzer = 0") + client2.expect(prompt) client1.send("CREATE DATABASE IF NOT EXISTS 01062_window_view_event_hop_watch_asc") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py index 7f00130b184..b8d5ff02d37 100755 --- a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py +++ b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py @@ -27,6 +27,8 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) + client2.send("SET allow_experimental_analyzer = 0") + client2.expect(prompt) client1.send( "CREATE DATABASE IF NOT EXISTS 01065_window_view_event_hop_watch_bounded" diff --git a/tests/queries/0_stateless/01070_window_view_watch_events.py b/tests/queries/0_stateless/01070_window_view_watch_events.py index 8aeff041cc1..1cf7678a014 100755 --- a/tests/queries/0_stateless/01070_window_view_watch_events.py +++ b/tests/queries/0_stateless/01070_window_view_watch_events.py @@ -28,6 +28,8 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) + client2.send("SET allow_experimental_analyzer = 0") + client2.expect(prompt) client1.send("CREATE DATABASE IF NOT EXISTS 01070_window_view_watch_events") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01078_window_view_alter_query_watch.py b/tests/queries/0_stateless/01078_window_view_alter_query_watch.py index c32e508c5a5..3f3dfe0cda8 100755 --- a/tests/queries/0_stateless/01078_window_view_alter_query_watch.py +++ b/tests/queries/0_stateless/01078_window_view_alter_query_watch.py @@ -28,10 +28,14 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) + client2.send("SET allow_experimental_analyzer = 0") + client2.expect(prompt) client3.send("SET allow_experimental_window_view = 1") client3.expect(prompt) client3.send("SET window_view_heartbeat_interval = 1") client3.expect(prompt) + client3.send("SET allow_experimental_analyzer = 0") + client3.expect(prompt) client1.send("CREATE DATABASE IF NOT EXISTS 01078_window_view_alter_query_watch") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01082_window_view_watch_limit.py b/tests/queries/0_stateless/01082_window_view_watch_limit.py index 12c8d295591..9938ebcab98 100755 --- a/tests/queries/0_stateless/01082_window_view_watch_limit.py +++ b/tests/queries/0_stateless/01082_window_view_watch_limit.py @@ -27,6 +27,8 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) + client2.send("SET allow_experimental_analyzer = 0") + client2.expect(prompt) client1.send("CREATE DATABASE IF NOT EXISTS 01082_window_view_watch_limit") client1.expect(prompt) From 7dacd8aa842b94811b111c35cc2b21ae6564ba54 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 7 Apr 2024 14:46:55 +0000 Subject: [PATCH 1057/1165] Incorporate review feedback --- src/Interpreters/Cache/QueryCache.cpp | 37 +++++++++++++------ .../02494_query_cache_system_tables.sql | 3 ++ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index 8a76378f14c..67fcdb8159c 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include #include #include /// chassert @@ -69,20 +71,31 @@ struct HasSystemTablesMatcher if (data.has_system_tables) return; - if (const auto * identifier = node->as()) + String database_table; /// or whatever else we get, e.g. just a table + + /// SELECT [...] FROM + if (const auto * table_identifier = node->as()) { - StorageID storage_id = identifier->getTableId(); - if (!storage_id.hasDatabase()) - /// The common case that a database name was not explicitly specified in the SQL. However, isPredefinedTable() is AST-based - /// and assumes that a database name was specified. This bites us in this edge situation: - /// USE SYSTEM; - /// SELECT * FROM PROCESSES; -- instead of SYSTEM.PROCESSES - /// In this case, don't call isPredefinedTable() (to avoid exceptions) and accept that the behavior is not 100% kosher. - return; - bool is_predefined_table = DatabaseCatalog::instance().isPredefinedTable(storage_id); - if (is_predefined_table) - data.has_system_tables = true; + database_table = table_identifier->name(); } + /// SELECT [...] FROM clusterAllReplicas(,
) + else if (const auto * identifier = node->as()) + { + database_table = identifier->name(); + } + /// Handle SELECT [...] FROM clusterAllReplicas(, '
') + else if (const auto * literal = node->as()) + { + const auto & value = literal->value; /// (*) + database_table = applyVisitor(FieldVisitorDump(), value); + } + + /// (*) returns table in quotes, so we can't use .starts_with() for matching + static const re2::RE2 is_system_table(String(DatabaseCatalog::TEMPORARY_DATABASE) + + "|" + DatabaseCatalog::SYSTEM_DATABASE + + "|" + DatabaseCatalog::INFORMATION_SCHEMA + + "|" + DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE); + data.has_system_tables = re2::RE2::PartialMatch(database_table, is_system_table); } }; diff --git a/tests/queries/0_stateless/02494_query_cache_system_tables.sql b/tests/queries/0_stateless/02494_query_cache_system_tables.sql index 89ea9d2b6b7..935011a6bb0 100644 --- a/tests/queries/0_stateless/02494_query_cache_system_tables.sql +++ b/tests/queries/0_stateless/02494_query_cache_system_tables.sql @@ -44,5 +44,8 @@ SELECT * SETTINGS use_query_cache = 1; SELECT * FROM information_schema.tables SETTINGS use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } SELECT * FROM INFORMATION_SCHEMA.TABLES SETTINGS use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } +SELECT * FROM clusterAllReplicas('test_shard_localhost', system.one) SETTINGS use_query_cache = 1; -- {serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } +SELECT * FROM clusterAllReplicas('test_shard_localhost', 'system.one') SETTINGS use_query_cache = 1; -- {serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } + -- Cleanup SYSTEM DROP QUERY CACHE; From 285dbc39f7b3c7c82122a3ecf908bba47ecc3132 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 7 Apr 2024 17:12:52 +0000 Subject: [PATCH 1058/1165] Fix tests --- .../0_stateless/02494_query_cache_eligible_queries.sql | 1 + tests/queries/0_stateless/02494_query_cache_explain.sql | 1 + .../queries/0_stateless/02494_query_cache_sparse_columns.sql | 5 ++--- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql b/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql index 5c45ee8aedd..f7ddb8f8bda 100644 --- a/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql +++ b/tests/queries/0_stateless/02494_query_cache_eligible_queries.sql @@ -7,6 +7,7 @@ DROP TABLE IF EXISTS eligible_test2; -- enable query cache session-wide but also force it individually in each of below statements SET use_query_cache = true; +SET query_cache_system_table_handling = 'save'; -- check that SELECT statements create entries in the query cache ... SELECT 1 SETTINGS use_query_cache = true; diff --git a/tests/queries/0_stateless/02494_query_cache_explain.sql b/tests/queries/0_stateless/02494_query_cache_explain.sql index d12938181c2..bf376b47fdb 100644 --- a/tests/queries/0_stateless/02494_query_cache_explain.sql +++ b/tests/queries/0_stateless/02494_query_cache_explain.sql @@ -2,6 +2,7 @@ -- Tag no-parallel: Messes with internal cache SET allow_experimental_analyzer = 1; +SET query_cache_system_table_handling = 'save'; SYSTEM DROP QUERY CACHE; diff --git a/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql b/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql index 4344d139d60..6266996ac2e 100644 --- a/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql +++ b/tests/queries/0_stateless/02494_query_cache_sparse_columns.sql @@ -12,11 +12,10 @@ SYSTEM STOP MERGES t_cache_sparse; INSERT INTO t_cache_sparse SELECT number, number FROM numbers(10000); INSERT INTO t_cache_sparse SELECT number, 0 FROM numbers(10000); -SET use_query_cache = 1; SET max_threads = 1; -SELECT v FROM t_cache_sparse FORMAT Null; -SELECT v FROM t_cache_sparse FORMAT Null; +SELECT v FROM t_cache_sparse SETTINGS use_query_cache = 1, max_threads = 1 FORMAT Null; +SELECT v FROM t_cache_sparse SETTINGS use_query_cache = 1, max_threads = 1 FORMAT Null; SELECT count() FROM system.query_cache WHERE query LIKE 'SELECT v FROM t_cache_sparse%'; DROP TABLE t_cache_sparse; From bbe8e2d751e2445afc07d97e18625d53868b2235 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 7 Apr 2024 20:35:27 +0300 Subject: [PATCH 1059/1165] Arrow schema to ClickHouse schema Nullable fix --- .../Formats/Impl/ArrowBlockInputFormat.cpp | 2 +- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 365 ++++++++++++------ .../Formats/Impl/ArrowColumnToCHColumn.h | 23 +- .../Formats/Impl/ORCBlockInputFormat.cpp | 4 +- .../Formats/Impl/ParquetBlockInputFormat.cpp | 2 +- .../DataLakes/DeltaLakeMetadataParser.cpp | 3 +- 6 files changed, 264 insertions(+), 135 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index 206e244c75f..fc9a827be66 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -86,7 +86,7 @@ Chunk ArrowBlockInputFormat::read() /// If defaults_for_omitted_fields is true, calculate the default values from default expression for omitted fields. /// Otherwise fill the missing columns with zero values of its type. BlockMissingValues * block_missing_values_ptr = format_settings.defaults_for_omitted_fields ? &block_missing_values : nullptr; - arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result, (*table_result)->num_rows(), block_missing_values_ptr); + res = arrow_column_to_ch_column->arrowTableToCHChunk(*table_result, (*table_result)->num_rows(), block_missing_values_ptr); /// There is no easy way to get original record batch size from Arrow metadata. /// Let's just use the number of bytes read from read buffer. diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 65704c85026..ec2d17d73cb 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -78,7 +78,7 @@ namespace ErrorCodes /// Inserts numeric data right into internal column data to reduce an overhead template > -static ColumnWithTypeAndName readColumnWithNumericData(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithNumericData(const std::shared_ptr & arrow_column, const String & column_name) { auto internal_type = std::make_shared>(); auto internal_column = internal_type->createColumn(); @@ -103,7 +103,7 @@ static ColumnWithTypeAndName readColumnWithNumericData(std::shared_ptr -static ColumnWithTypeAndName readColumnWithStringData(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithStringData(const std::shared_ptr & arrow_column, const String & column_name) { auto internal_type = std::make_shared(); auto internal_column = internal_type->createColumn(); @@ -147,7 +147,7 @@ static ColumnWithTypeAndName readColumnWithStringData(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithFixedStringData(const std::shared_ptr & arrow_column, const String & column_name) { const auto * fixed_type = assert_cast(arrow_column->type().get()); size_t fixed_len = fixed_type->byte_width(); @@ -166,7 +166,7 @@ static ColumnWithTypeAndName readColumnWithFixedStringData(std::shared_ptr -static ColumnWithTypeAndName readColumnWithBigIntegerFromFixedBinaryData(std::shared_ptr & arrow_column, const String & column_name, const DataTypePtr & column_type) +static ColumnWithTypeAndName readColumnWithBigIntegerFromFixedBinaryData(const std::shared_ptr & arrow_column, const String & column_name, const DataTypePtr & column_type) { const auto * fixed_type = assert_cast(arrow_column->type().get()); size_t fixed_len = fixed_type->byte_width(); @@ -193,7 +193,7 @@ static ColumnWithTypeAndName readColumnWithBigIntegerFromFixedBinaryData(std::sh } template -static ColumnWithTypeAndName readColumnWithBigNumberFromBinaryData(std::shared_ptr & arrow_column, const String & column_name, const DataTypePtr & column_type) +static ColumnWithTypeAndName readColumnWithBigNumberFromBinaryData(const std::shared_ptr & arrow_column, const String & column_name, const DataTypePtr & column_type) { size_t total_size = 0; for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) @@ -229,7 +229,7 @@ static ColumnWithTypeAndName readColumnWithBigNumberFromBinaryData(std::shared_p return {std::move(internal_column), column_type, column_name}; } -static ColumnWithTypeAndName readColumnWithBooleanData(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithBooleanData(const std::shared_ptr & arrow_column, const String & column_name) { auto internal_type = DataTypeFactory::instance().get("Bool"); auto internal_column = internal_type->createColumn(); @@ -248,7 +248,7 @@ static ColumnWithTypeAndName readColumnWithBooleanData(std::shared_ptr & arrow_column, const String & column_name, +static ColumnWithTypeAndName readColumnWithDate32Data(const std::shared_ptr & arrow_column, const String & column_name, const DataTypePtr & type_hint, FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior) { DataTypePtr internal_type; @@ -310,7 +310,7 @@ static ColumnWithTypeAndName readColumnWithDate32Data(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithDate64Data(const std::shared_ptr & arrow_column, const String & column_name) { auto internal_type = std::make_shared(); auto internal_column = internal_type->createColumn(); @@ -329,7 +329,7 @@ static ColumnWithTypeAndName readColumnWithDate64Data(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithTimestampData(const std::shared_ptr & arrow_column, const String & column_name) { const auto & arrow_type = static_cast(*(arrow_column->type())); const UInt8 scale = arrow_type.unit() * 3; @@ -350,7 +350,7 @@ static ColumnWithTypeAndName readColumnWithTimestampData(std::shared_ptr -static ColumnWithTypeAndName readColumnWithTimeData(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithTimeData(const std::shared_ptr & arrow_column, const String & column_name) { const auto & arrow_type = static_cast(*(arrow_column->type())); const UInt8 scale = arrow_type.unit() * 3; @@ -373,18 +373,18 @@ static ColumnWithTypeAndName readColumnWithTimeData(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithTime32Data(const std::shared_ptr & arrow_column, const String & column_name) { return readColumnWithTimeData(arrow_column, column_name); } -static ColumnWithTypeAndName readColumnWithTime64Data(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithTime64Data(const std::shared_ptr & arrow_column, const String & column_name) { return readColumnWithTimeData(arrow_column, column_name); } template -static ColumnWithTypeAndName readColumnWithDecimalDataImpl(std::shared_ptr & arrow_column, const String & column_name, DataTypePtr internal_type) +static ColumnWithTypeAndName readColumnWithDecimalDataImpl(const std::shared_ptr & arrow_column, const String & column_name, DataTypePtr internal_type) { auto internal_column = internal_type->createColumn(); auto & column = assert_cast &>(*internal_column); @@ -403,7 +403,7 @@ static ColumnWithTypeAndName readColumnWithDecimalDataImpl(std::shared_ptr -static ColumnWithTypeAndName readColumnWithDecimalData(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readColumnWithDecimalData(const std::shared_ptr & arrow_column, const String & column_name) { const auto * arrow_decimal_type = static_cast(arrow_column->type().get()); size_t precision = arrow_decimal_type->precision(); @@ -418,7 +418,7 @@ static ColumnWithTypeAndName readColumnWithDecimalData(std::shared_ptr & arrow_column) +static ColumnPtr readByteMapFromArrowColumn(const std::shared_ptr & arrow_column) { if (!arrow_column->null_count()) return ColumnUInt8::create(arrow_column->length(), 0); @@ -453,7 +453,7 @@ struct ArrowOffsetArray }; template -static ColumnPtr readOffsetsFromArrowListColumn(std::shared_ptr & arrow_column) +static ColumnPtr readOffsetsFromArrowListColumn(const std::shared_ptr & arrow_column) { auto offsets_column = ColumnUInt64::create(); ColumnArray::Offsets & offsets_data = assert_cast &>(*offsets_column).getData(); @@ -463,7 +463,7 @@ static ColumnPtr readOffsetsFromArrowListColumn(std::shared_ptr(*(arrow_column->chunk(chunk_i))); auto arrow_offsets_array = list_chunk.offsets(); - auto & arrow_offsets = dynamic_cast::type &>(*arrow_offsets_array); + auto & arrow_offsets = dynamic_cast::type &>(*arrow_offsets_array); /* * CH uses element size as "offsets", while arrow uses actual offsets as offsets. @@ -620,7 +620,7 @@ static ColumnPtr readColumnWithIndexesData(std::shared_ptr } template -static std::shared_ptr getNestedArrowColumn(std::shared_ptr & arrow_column) +static std::shared_ptr getNestedArrowColumn(const std::shared_ptr & arrow_column) { arrow::ArrayVector array_vector; array_vector.reserve(arrow_column->num_chunks()); @@ -648,7 +648,7 @@ static std::shared_ptr getNestedArrowColumn(std::shared_ptr return std::make_shared(array_vector); } -static ColumnWithTypeAndName readIPv6ColumnFromBinaryData(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readIPv6ColumnFromBinaryData(const std::shared_ptr & arrow_column, const String & column_name) { size_t total_size = 0; for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) @@ -684,7 +684,7 @@ static ColumnWithTypeAndName readIPv6ColumnFromBinaryData(std::shared_ptr & arrow_column, const String & column_name) +static ColumnWithTypeAndName readIPv4ColumnWithInt32Data(const std::shared_ptr & arrow_column, const String & column_name) { auto internal_type = std::make_shared(); auto internal_column = internal_type->createColumn(); @@ -705,35 +705,31 @@ static ColumnWithTypeAndName readIPv4ColumnWithInt32Data(std::shared_ptr & arrow_column, - const std::string & column_name, - const std::string & format_name, - bool is_nullable, - std::unordered_map & dictionary_infos, - bool allow_null_type, - bool skip_columns_with_unsupported_types, - bool & skipped, - FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior = FormatSettings::DateTimeOverflowBehavior::Ignore, - DataTypePtr type_hint = nullptr, - bool is_map_nested = false) +struct ReadColumnFromArrowColumnSettings { - if (!is_nullable && (arrow_column->null_count() || (type_hint && type_hint->isNullable())) && arrow_column->type()->id() != arrow::Type::LIST - && arrow_column->type()->id() != arrow::Type::MAP && arrow_column->type()->id() != arrow::Type::STRUCT && - arrow_column->type()->id() != arrow::Type::DICTIONARY) - { - DataTypePtr nested_type_hint; - if (type_hint) - nested_type_hint = removeNullable(type_hint); - auto nested_column = readColumnFromArrowColumn(arrow_column, column_name, format_name, true, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, date_time_overflow_behavior, nested_type_hint); - if (skipped) - return {}; - auto nullmap_column = readByteMapFromArrowColumn(arrow_column); - auto nullable_type = std::make_shared(std::move(nested_column.type)); - auto nullable_column = ColumnNullable::create(nested_column.column, nullmap_column); - return {std::move(nullable_column), std::move(nullable_type), column_name}; - } + std::string format_name; + FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior; + bool allow_arrow_null_type; + bool skip_columns_with_unsupported_types; +}; +static ColumnWithTypeAndName readColumnFromArrowColumn( + const std::shared_ptr & arrow_column, + std::string column_name, + std::unordered_map dictionary_infos, + DataTypePtr type_hint, + bool is_nullable_column, + bool is_map_nested_column, + const ReadColumnFromArrowColumnSettings & settings); + +static ColumnWithTypeAndName readNonNullableColumnFromArrowColumn( + const std::shared_ptr & arrow_column, + std::string column_name, + std::unordered_map dictionary_infos, + DataTypePtr type_hint, + bool is_map_nested_column, + const ReadColumnFromArrowColumnSettings & settings) +{ switch (arrow_column->type()->id()) { case arrow::Type::STRING: @@ -790,7 +786,7 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( case arrow::Type::BOOL: return readColumnWithBooleanData(arrow_column, column_name); case arrow::Type::DATE32: - return readColumnWithDate32Data(arrow_column, column_name, type_hint, date_time_overflow_behavior); + return readColumnWithDate32Data(arrow_column, column_name, type_hint, settings.date_time_overflow_behavior); case arrow::Type::DATE64: return readColumnWithDate64Data(arrow_column, column_name); // ClickHouse writes Date as arrow UINT16 and DateTime as arrow UINT32, @@ -837,9 +833,16 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( key_type_hint = map_type_hint->getKeyType(); } } + auto arrow_nested_column = getNestedArrowColumn(arrow_column); - auto nested_column = readColumnFromArrowColumn(arrow_nested_column, column_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, date_time_overflow_behavior, nested_type_hint, true); - if (skipped) + auto nested_column = readColumnFromArrowColumn(arrow_nested_column, + column_name, + dictionary_infos, + nested_type_hint, + false /*is_nullable_column*/, + true /*is_map_nested_column*/, + settings); + if (!nested_column.column) return {}; auto offsets_column = readOffsetsFromArrowListColumn(arrow_column); @@ -866,7 +869,7 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( case arrow::Type::LIST: case arrow::Type::LARGE_LIST: { - bool is_large = arrow_column->type()->id() == arrow::Type::LARGE_LIST; + bool is_large_list = arrow_column->type()->id() == arrow::Type::LARGE_LIST; DataTypePtr nested_type_hint; if (type_hint) { @@ -874,12 +877,33 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( if (array_type_hint) nested_type_hint = array_type_hint->getNestedType(); } - auto arrow_nested_column = is_large ? getNestedArrowColumn(arrow_column) : getNestedArrowColumn(arrow_column); - auto nested_column = readColumnFromArrowColumn(arrow_nested_column, column_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, date_time_overflow_behavior, nested_type_hint); - if (skipped) + + bool is_nested_nullable_column = false; + if (is_large_list) + { + auto * arrow_large_list_type = assert_cast(arrow_column->type().get()); + is_nested_nullable_column = arrow_large_list_type->value_field()->nullable(); + } + else + { + auto * arrow_list_type = assert_cast(arrow_column->type().get()); + is_nested_nullable_column = arrow_list_type->value_field()->nullable(); + } + + auto arrow_nested_column = is_large_list ? getNestedArrowColumn(arrow_column) : getNestedArrowColumn(arrow_column); + auto nested_column = readColumnFromArrowColumn(arrow_nested_column, + column_name, + dictionary_infos, + nested_type_hint, + is_nested_nullable_column, + false /*is_map_nested_column*/, + settings); + if (!nested_column.column) return {}; - auto offsets_column = is_large ? readOffsetsFromArrowListColumn(arrow_column) : readOffsetsFromArrowListColumn(arrow_column); + + auto offsets_column = is_large_list ? readOffsetsFromArrowListColumn(arrow_column) : readOffsetsFromArrowListColumn(arrow_column); auto array_column = ColumnArray::create(nested_column.column, offsets_column); + DataTypePtr array_type; /// If type hint is Nested, we should return Nested type, /// because we differentiate Nested and simple Array(Tuple) @@ -913,11 +937,13 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( for (int i = 0; i != arrow_struct_type->num_fields(); ++i) { - auto field_name = arrow_struct_type->field(i)->name(); + const auto & field = arrow_struct_type->field(i); + const auto & field_name = field->name(); + DataTypePtr nested_type_hint; if (tuple_type_hint) { - if (tuple_type_hint->haveExplicitNames() && !is_map_nested) + if (tuple_type_hint->haveExplicitNames() && !is_map_nested_column) { auto pos = tuple_type_hint->tryGetPositionByName(field_name); if (pos) @@ -926,13 +952,21 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( else if (size_t(i) < tuple_type_hint->getElements().size()) nested_type_hint = tuple_type_hint->getElement(i); } + auto nested_arrow_column = std::make_shared(nested_arrow_columns[i]); - auto element = readColumnFromArrowColumn(nested_arrow_column, field_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, date_time_overflow_behavior, nested_type_hint); - if (skipped) + auto column_with_type_and_name = readColumnFromArrowColumn(nested_arrow_column, + field_name, + dictionary_infos, + nested_type_hint, + field->nullable(), + false /*is_map_nested_column*/, + settings); + if (!column_with_type_and_name.column) return {}; - tuple_elements.emplace_back(std::move(element.column)); - tuple_types.emplace_back(std::move(element.type)); - tuple_names.emplace_back(std::move(element.name)); + + tuple_elements.emplace_back(std::move(column_with_type_and_name.column)); + tuple_types.emplace_back(std::move(column_with_type_and_name.type)); + tuple_names.emplace_back(std::move(column_with_type_and_name.name)); } auto tuple_column = ColumnTuple::create(std::move(tuple_elements)); @@ -953,8 +987,19 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( arrow::DictionaryArray & dict_chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); dict_array.emplace_back(dict_chunk.dictionary()); } + auto arrow_dict_column = std::make_shared(dict_array); - auto dict_column = readColumnFromArrowColumn(arrow_dict_column, column_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, date_time_overflow_behavior); + auto dict_column = readColumnFromArrowColumn(arrow_dict_column, + column_name, + dictionary_infos, + nullptr /*nested_type_hint*/, + false /*is_nullable_column*/, + false /*is_map_nested_column*/, + settings); + + if (!dict_column.column) + return {}; + for (size_t i = 0; i != dict_column.column->size(); ++i) { if (dict_column.column->isDefaultAt(i)) @@ -963,6 +1008,7 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( break; } } + auto lc_type = std::make_shared(is_lc_nullable ? makeNullable(dict_column.type) : dict_column.type); auto tmp_lc_column = lc_type->createColumn(); auto tmp_dict_column = IColumn::mutate(assert_cast(tmp_lc_column.get())->getDictionaryPtr()); @@ -1002,7 +1048,7 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( // TODO: read UUID as a string? case arrow::Type::NA: { - if (allow_null_type) + if (settings.allow_arrow_null_type) { auto type = std::make_shared(); auto column = ColumnNothing::create(arrow_column->length()); @@ -1012,11 +1058,8 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( } default: { - if (skip_columns_with_unsupported_types) - { - skipped = true; + if (settings.skip_columns_with_unsupported_types) return {}; - } throw Exception( ErrorCodes::UNKNOWN_TYPE, @@ -1024,14 +1067,59 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( "If it happens during schema inference and you want to skip columns with " "unsupported types, you can enable setting input_format_{}" "_skip_columns_with_unsupported_types_in_schema_inference", - format_name, + settings.format_name, arrow_column->type()->name(), column_name, - boost::algorithm::to_lower_copy(format_name)); + boost::algorithm::to_lower_copy(settings.format_name)); } } } +static ColumnWithTypeAndName readColumnFromArrowColumn( + const std::shared_ptr & arrow_column, + std::string column_name, + std::unordered_map dictionary_infos, + DataTypePtr type_hint, + bool is_nullable_column, + bool is_map_nested_column, + const ReadColumnFromArrowColumnSettings & settings) +{ + bool read_as_nullable_column = arrow_column->null_count() || is_nullable_column || (type_hint && type_hint->isNullable()); + if (read_as_nullable_column && + arrow_column->type()->id() != arrow::Type::LIST && + arrow_column->type()->id() != arrow::Type::LARGE_LIST && + arrow_column->type()->id() != arrow::Type::MAP && + arrow_column->type()->id() != arrow::Type::STRUCT && + arrow_column->type()->id() != arrow::Type::DICTIONARY) + { + DataTypePtr nested_type_hint; + if (type_hint) + nested_type_hint = removeNullable(type_hint); + + auto nested_column = readNonNullableColumnFromArrowColumn(arrow_column, + column_name, + dictionary_infos, + nested_type_hint, + is_map_nested_column, + settings); + + if (!nested_column.column) + return {}; + + auto nullmap_column = readByteMapFromArrowColumn(arrow_column); + auto nullable_type = std::make_shared(std::move(nested_column.type)); + auto nullable_column = ColumnNullable::create(nested_column.column, nullmap_column); + + return {std::move(nullable_column), std::move(nullable_type), column_name}; + } + + return readNonNullableColumnFromArrowColumn(arrow_column, + column_name, + dictionary_infos, + type_hint, + is_map_nested_column, + settings); +} // Creating CH header by arrow schema. Will be useful in task about inserting // data from file without knowing table structure. @@ -1042,44 +1130,56 @@ static void checkStatus(const arrow::Status & status, const String & column_name throw Exception{ErrorCodes::UNKNOWN_EXCEPTION, "Error with a {} column '{}': {}.", format_name, column_name, status.ToString()}; } +/// Create empty arrow column using specified field +static std::shared_ptr createArrowColumn(const std::shared_ptr & field, const String & format_name) +{ + arrow::MemoryPool * pool = arrow::default_memory_pool(); + std::unique_ptr array_builder; + arrow::Status status = MakeBuilder(pool, field->type(), &array_builder); + checkStatus(status, field->name(), format_name); + + std::shared_ptr arrow_array; + status = array_builder->Finish(&arrow_array); + checkStatus(status, field->name(), format_name); + + return std::make_shared(arrow::ArrayVector{arrow_array}); +} Block ArrowColumnToCHColumn::arrowSchemaToCHHeader( - const arrow::Schema & schema, const std::string & format_name, - bool skip_columns_with_unsupported_types, const Block * hint_header, bool ignore_case) + const arrow::Schema & schema, + const std::string & format_name, + bool skip_columns_with_unsupported_types) { + ReadColumnFromArrowColumnSettings settings + { + .format_name = format_name, + .date_time_overflow_behavior = FormatSettings::DateTimeOverflowBehavior::Ignore, + .allow_arrow_null_type = false, + .skip_columns_with_unsupported_types = skip_columns_with_unsupported_types + }; + ColumnsWithTypeAndName sample_columns; - std::unordered_set nested_table_names; - if (hint_header) - nested_table_names = Nested::getAllTableNames(*hint_header, ignore_case); for (const auto & field : schema.fields()) { - if (hint_header && !hint_header->has(field->name(), ignore_case) - && !nested_table_names.contains(ignore_case ? boost::to_lower_copy(field->name()) : field->name())) - continue; - /// Create empty arrow column by it's type and convert it to ClickHouse column. - arrow::MemoryPool * pool = arrow::default_memory_pool(); - std::unique_ptr array_builder; - arrow::Status status = MakeBuilder(pool, field->type(), &array_builder); - checkStatus(status, field->name(), format_name); + auto arrow_column = createArrowColumn(field, format_name); - std::shared_ptr arrow_array; - status = array_builder->Finish(&arrow_array); - checkStatus(status, field->name(), format_name); - - arrow::ArrayVector array_vector = {arrow_array}; - auto arrow_column = std::make_shared(array_vector); std::unordered_map dict_infos; - bool skipped = false; - bool allow_null_type = false; - if (hint_header && hint_header->has(field->name()) && hint_header->getByName(field->name()).type->isNullable()) - allow_null_type = true; - ColumnWithTypeAndName sample_column = readColumnFromArrowColumn( - arrow_column, field->name(), format_name, false, dict_infos, allow_null_type, skip_columns_with_unsupported_types, skipped); - if (!skipped) + + auto sample_column = readColumnFromArrowColumn( + arrow_column, + field->name(), + dict_infos, + nullptr /*nested_type_hint*/, + field->nullable() /*is_nullable_column*/, + false /*is_map_nested_column*/, + settings); + + if (sample_column.column) sample_columns.emplace_back(std::move(sample_column)); } + return Block(std::move(sample_columns)); } @@ -1101,30 +1201,43 @@ ArrowColumnToCHColumn::ArrowColumnToCHColumn( { } -void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, size_t num_rows, BlockMissingValues * block_missing_values) +Chunk ArrowColumnToCHColumn::arrowTableToCHChunk(const std::shared_ptr & table, size_t num_rows, BlockMissingValues * block_missing_values) { - NameToColumnPtr name_to_column_ptr; + NameToArrowColumn name_to_arrow_column; + for (auto column_name : table->ColumnNames()) { - std::shared_ptr arrow_column = table->GetColumnByName(column_name); + auto arrow_column = table->GetColumnByName(column_name); if (!arrow_column) throw Exception(ErrorCodes::DUPLICATE_COLUMN, "Column '{}' is duplicated", column_name); + auto arrow_field = table->schema()->GetFieldByName(column_name); + if (case_insensitive_matching) boost::to_lower(column_name); - name_to_column_ptr[std::move(column_name)] = arrow_column; + + name_to_arrow_column[std::move(column_name)] = {std::move(arrow_column), std::move(arrow_field)}; } - arrowColumnsToCHChunk(res, name_to_column_ptr, num_rows, block_missing_values); + return arrowColumnsToCHChunk(name_to_arrow_column, num_rows, block_missing_values); } -void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & name_to_column_ptr, size_t num_rows, BlockMissingValues * block_missing_values) +Chunk ArrowColumnToCHColumn::arrowColumnsToCHChunk(const NameToArrowColumn & name_to_arrow_column, size_t num_rows, BlockMissingValues * block_missing_values) { - Columns columns_list; - columns_list.reserve(header.columns()); + ReadColumnFromArrowColumnSettings settings + { + .format_name = format_name, + .date_time_overflow_behavior = date_time_overflow_behavior, + .allow_arrow_null_type = true, + .skip_columns_with_unsupported_types = false + }; + + Columns columns; + columns.reserve(header.columns()); + std::unordered_map>> nested_tables; - bool skipped = false; - for (size_t column_i = 0, columns = header.columns(); column_i < columns; ++column_i) + + for (size_t column_i = 0, header_columns = header.columns(); column_i < header_columns; ++column_i) { const ColumnWithTypeAndName & header_column = header.getByPosition(column_i); @@ -1133,15 +1246,17 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & boost::to_lower(search_column_name); ColumnWithTypeAndName column; - if (!name_to_column_ptr.contains(search_column_name)) + if (!name_to_arrow_column.contains(search_column_name)) { bool read_from_nested = false; + /// Check if it's a subcolumn from some struct. String nested_table_name = Nested::extractTableName(header_column.name); String search_nested_table_name = nested_table_name; if (case_insensitive_matching) boost::to_lower(search_nested_table_name); - if (name_to_column_ptr.contains(search_nested_table_name)) + + if (name_to_arrow_column.contains(search_nested_table_name)) { if (!nested_tables.contains(search_nested_table_name)) { @@ -1153,10 +1268,19 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & } auto nested_table_type = Nested::collect(nested_columns).front().type; - std::shared_ptr arrow_column = name_to_column_ptr[search_nested_table_name]; - ColumnsWithTypeAndName cols = { - readColumnFromArrowColumn(arrow_column, nested_table_name, format_name, false, dictionary_infos, true, false, - skipped, date_time_overflow_behavior, nested_table_type)}; + const auto & arrow_column = name_to_arrow_column.find(search_nested_table_name)->second; + + ColumnsWithTypeAndName cols = + { + readColumnFromArrowColumn(arrow_column.column, + nested_table_name, + dictionary_infos, + nested_table_type, + arrow_column.field->nullable() /*is_nullable_column*/, + false /*is_map_nested_column*/, + settings) + }; + BlockPtr block_ptr = std::make_shared(cols); auto column_extractor = std::make_shared(*block_ptr, case_insensitive_matching); nested_tables[search_nested_table_name] = {block_ptr, column_extractor}; @@ -1180,7 +1304,7 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & column.name = header_column.name; column.type = header_column.type; column.column = header_column.column->cloneResized(num_rows); - columns_list.push_back(std::move(column.column)); + columns.push_back(std::move(column.column)); if (block_missing_values) block_missing_values->setBits(column_i, num_rows); continue; @@ -1189,9 +1313,14 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & } else { - auto arrow_column = name_to_column_ptr[search_column_name]; - column = readColumnFromArrowColumn( - arrow_column, header_column.name, format_name, false, dictionary_infos, true, false, skipped, date_time_overflow_behavior, header_column.type); + const auto & arrow_column = name_to_arrow_column.find(search_column_name)->second; + column = readColumnFromArrowColumn(arrow_column.column, + header_column.name, + dictionary_infos, + header_column.type, + arrow_column.field->nullable(), + false /*is_map_nested_column*/, + settings); } if (null_as_default) @@ -1216,10 +1345,10 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & } column.type = header_column.type; - columns_list.push_back(std::move(column.column)); + columns.push_back(std::move(column.column)); } - res.setColumns(columns_list, num_rows); + return Chunk(std::move(columns), num_rows); } } diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index 079e0374917..27e9afdf763 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -19,8 +19,6 @@ class Chunk; class ArrowColumnToCHColumn { public: - using NameToColumnPtr = std::unordered_map>; - ArrowColumnToCHColumn( const Block & header_, const std::string & format_name_, @@ -30,18 +28,13 @@ public: bool case_insensitive_matching_ = false, bool is_stream_ = false); - void arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, size_t num_rows, BlockMissingValues * block_missing_values = nullptr); + Chunk arrowTableToCHChunk(const std::shared_ptr & table, size_t num_rows, BlockMissingValues * block_missing_values = nullptr); - void arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & name_to_column_ptr, size_t num_rows, BlockMissingValues * block_missing_values = nullptr); - - /// Transform arrow schema to ClickHouse header. If hint_header is provided, - /// we will skip columns in schema that are not in hint_header. + /// Transform arrow schema to ClickHouse header static Block arrowSchemaToCHHeader( const arrow::Schema & schema, const std::string & format_name, - bool skip_columns_with_unsupported_types = false, - const Block * hint_header = nullptr, - bool ignore_case = false); + bool skip_columns_with_unsupported_types = false); struct DictionaryInfo { @@ -52,6 +45,16 @@ public: private: + struct ArrowColumn + { + std::shared_ptr column; + std::shared_ptr field; + }; + + using NameToArrowColumn = std::unordered_map; + + Chunk arrowColumnsToCHChunk(const NameToArrowColumn & name_to_arrow_column, size_t num_rows, BlockMissingValues * block_missing_values); + const Block & header; const std::string format_name; /// If false, throw exception if some columns in header not exists in arrow table. diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index a41eacf26b7..aa83b87b2d2 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -71,12 +71,10 @@ Chunk ORCBlockInputFormat::read() approx_bytes_read_for_chunk = file_reader->GetRawORCReader()->getStripe(stripe_current)->getDataLength(); ++stripe_current; - Chunk res; /// If defaults_for_omitted_fields is true, calculate the default values from default expression for omitted fields. /// Otherwise fill the missing columns with zero values of its type. BlockMissingValues * block_missing_values_ptr = format_settings.defaults_for_omitted_fields ? &block_missing_values : nullptr; - arrow_column_to_ch_column->arrowTableToCHChunk(res, table, num_rows, block_missing_values_ptr); - return res; + return arrow_column_to_ch_column->arrowTableToCHChunk(table, num_rows, block_missing_values_ptr); } void ORCBlockInputFormat::resetParser() diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 62e576d4953..d41cb3447de 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -601,7 +601,7 @@ void ParquetBlockInputFormat::decodeOneChunk(size_t row_group_batch_idx, std::un /// If defaults_for_omitted_fields is true, calculate the default values from default expression for omitted fields. /// Otherwise fill the missing columns with zero values of its type. BlockMissingValues * block_missing_values_ptr = format_settings.defaults_for_omitted_fields ? &res.block_missing_values : nullptr; - row_group_batch.arrow_column_to_ch_column->arrowTableToCHChunk(res.chunk, *tmp_table, (*tmp_table)->num_rows(), block_missing_values_ptr); + res.chunk = row_group_batch.arrow_column_to_ch_column->arrowTableToCHChunk(*tmp_table, (*tmp_table)->num_rows(), block_missing_values_ptr); lock.lock(); diff --git a/src/Storages/DataLakes/DeltaLakeMetadataParser.cpp b/src/Storages/DataLakes/DeltaLakeMetadataParser.cpp index 3584f137225..14a912a180d 100644 --- a/src/Storages/DataLakes/DeltaLakeMetadataParser.cpp +++ b/src/Storages/DataLakes/DeltaLakeMetadataParser.cpp @@ -282,11 +282,10 @@ struct DeltaLakeMetadataParser::Impl format_settings.date_time_overflow_behavior, /* case_insensitive_column_matching */false); - Chunk res; std::shared_ptr table; THROW_ARROW_NOT_OK(reader->ReadTable(&table)); - column_reader.arrowTableToCHChunk(res, table, reader->parquet_reader()->metadata()->num_rows()); + Chunk res = column_reader.arrowTableToCHChunk(table, reader->parquet_reader()->metadata()->num_rows()); const auto & res_columns = res.getColumns(); if (res_columns.size() != 2) From 1e05d9ed3f4a960ca6a219514fd56fa13c644efc Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 7 Apr 2024 20:36:02 +0300 Subject: [PATCH 1060/1165] Added tests --- .../03036_parquet_arrow_nullable.reference | 40 ++++++++++++ .../03036_parquet_arrow_nullable.sh | 63 +++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 tests/queries/0_stateless/03036_parquet_arrow_nullable.reference create mode 100755 tests/queries/0_stateless/03036_parquet_arrow_nullable.sh diff --git a/tests/queries/0_stateless/03036_parquet_arrow_nullable.reference b/tests/queries/0_stateless/03036_parquet_arrow_nullable.reference new file mode 100644 index 00000000000..8820bb7cb9f --- /dev/null +++ b/tests/queries/0_stateless/03036_parquet_arrow_nullable.reference @@ -0,0 +1,40 @@ +Parquet +a UInt64 +a_nullable Nullable(UInt64) +Arrow +a UInt64 +a_nullable Nullable(UInt64) +Parquet +b Array(Nullable(UInt64)) +b_nullable Array(Nullable(UInt64)) +Arrow +b Array(Nullable(UInt64)) +b_nullable Array(Nullable(UInt64)) +Parquet +c Tuple(\n a UInt64,\n b String) +c_nullable Tuple(\n a Nullable(UInt64),\n b Nullable(String)) +Arrow +c Tuple(\n a UInt64,\n b String) +c_nullable Tuple(\n a Nullable(UInt64),\n b Nullable(String)) +Parquet +d Tuple(\n a UInt64,\n b Tuple(\n a UInt64,\n b String),\n d_nullable Tuple(\n a UInt64,\n b Tuple(\n a Nullable(UInt64),\n b Nullable(String)))) +Arrow +d Tuple(\n a UInt64,\n b Tuple(\n a UInt64,\n b String),\n d_nullable Tuple(\n a UInt64,\n b Tuple(\n a Nullable(UInt64),\n b Nullable(String)))) +Parquet +e Map(UInt64, Nullable(String)) +e_nullable Map(UInt64, Nullable(String)) +Arrow +e Map(UInt64, Nullable(String)) +e_nullable Map(UInt64, Nullable(String)) +Parquet +f Map(UInt64, Map(UInt64, Nullable(String))) +f_nullables Map(UInt64, Map(UInt64, Nullable(String))) +Arrow +f Map(UInt64, Map(UInt64, Nullable(String))) +f_nullables Map(UInt64, Map(UInt64, Nullable(String))) +Parquet +g String +g_nullable Nullable(String) +Arrow +g LowCardinality(String) +g_nullable LowCardinality(String) diff --git a/tests/queries/0_stateless/03036_parquet_arrow_nullable.sh b/tests/queries/0_stateless/03036_parquet_arrow_nullable.sh new file mode 100755 index 00000000000..bdd641e2b94 --- /dev/null +++ b/tests/queries/0_stateless/03036_parquet_arrow_nullable.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +DATA_FILE=$CLICKHOUSE_TEST_UNIQUE_NAME.data + +formats="Parquet Arrow" + +for format in $formats +do + echo $format + $CLICKHOUSE_LOCAL -q "select * from generateRandom('a UInt64, a_nullable Nullable(UInt64)', 42) limit 10 format $format" > $DATA_FILE + $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE') SETTINGS schema_inference_make_columns_nullable = 0" +done + +for format in $formats +do + echo $format + $CLICKHOUSE_LOCAL -q "select * from generateRandom('b Array(UInt64), b_nullable Array(Nullable(UInt64))', 42) limit 10 format $format" > $DATA_FILE + $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE') SETTINGS schema_inference_make_columns_nullable = 0" +done + +for format in $formats +do + echo $format + $CLICKHOUSE_LOCAL -q "select * from generateRandom('c Tuple(a UInt64, b String), c_nullable Tuple(a Nullable(UInt64), b Nullable(String))', 42) limit 10 format $format" > $DATA_FILE + $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE') SETTINGS schema_inference_make_columns_nullable = 0" +done + +for format in $formats +do + echo $format + $CLICKHOUSE_LOCAL -q "select * from generateRandom('d Tuple(a UInt64, b Tuple(a UInt64, b String), d_nullable Tuple(a UInt64, b Tuple(a Nullable(UInt64), b Nullable(String))))', 42) limit 10 format $format" > $DATA_FILE + $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE') SETTINGS schema_inference_make_columns_nullable = 0" +done + +for format in $formats +do + echo $format + $CLICKHOUSE_LOCAL -q "select * from generateRandom('e Map(UInt64, String), e_nullable Map(UInt64, Nullable(String))', 42) limit 10 format $format" > $DATA_FILE + $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE') SETTINGS schema_inference_make_columns_nullable = 0" +done + +for format in $formats +do + echo $format + $CLICKHOUSE_LOCAL -q "select * from generateRandom('f Map(UInt64, Map(UInt64, String)), f_nullables Map(UInt64, Map(UInt64, Nullable(String)))', 42) limit 10 format $format" > $DATA_FILE + $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE') SETTINGS schema_inference_make_columns_nullable = 0" +done + +for format in $formats +do + echo $format + $CLICKHOUSE_LOCAL -q "select * from generateRandom('g LowCardinality(String), g_nullable LowCardinality(Nullable(String))', 42) limit 10 settings output_format_arrow_low_cardinality_as_dictionary=1, allow_suspicious_low_cardinality_types=1 format $format" > $DATA_FILE + $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE') SETTINGS schema_inference_make_columns_nullable = 0" +done + +rm $DATA_FILE + From 373bf4968408e41392f5dc7e6791fc61143939f9 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 7 Apr 2024 20:05:41 +0200 Subject: [PATCH 1061/1165] fix --- src/Storages/WindowView/StorageWindowView.cpp | 2 +- .../queries/0_stateless/01069_window_view_proc_tumble_watch.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index f82f5b07910..04c26053dba 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -1710,7 +1710,7 @@ void StorageWindowView::throwIfWindowViewIsDisabled(ContextPtr local_context) co { if (disabled_due_to_analyzer || (local_context && local_context->getSettingsRef().allow_experimental_analyzer)) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Experimental WINDOW VIEW feature is not supported " - "with new infrastructure for query analysis (the setting 'allow_experimental_analyzer')"); + "in the current infrastructure for query analysis (the setting 'allow_experimental_analyzer')"); } void registerStorageWindowView(StorageFactory & factory) diff --git a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py index eb31b2ccbcf..21c2e831afc 100755 --- a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py +++ b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py @@ -28,6 +28,8 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) + client2.send("SET allow_experimental_analyzer = 0") + client2.expect(prompt) client1.send("CREATE DATABASE 01069_window_view_proc_tumble_watch") client1.expect(prompt) From 4b5db7357caf36ca797293ef1ec81df7c492e3cb Mon Sep 17 00:00:00 2001 From: Eduard Karacharov Date: Sun, 7 Apr 2024 20:48:59 +0300 Subject: [PATCH 1062/1165] docs: predefined query handler example fix --- docs/en/interfaces/http.md | 18 ++++++++++-------- docs/ru/interfaces/http.md | 18 ++++++++++-------- docs/zh/interfaces/http.md | 21 ++++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 4eeb19cefcf..bba5cde16f1 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -507,16 +507,18 @@ Example: ``` xml - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> GET TEST_HEADER_VALUE - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> predefined_query_handler - SELECT value FROM system.settings WHERE name = {name_1:String} - SELECT name, value FROM system.settings WHERE name = {name_2:String} + + SELECT name, value FROM system.settings + WHERE name IN ({name_1:String}, {name_2:String}) + @@ -524,13 +526,13 @@ Example: ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/query_param_with_url/1/max_threads/max_final_threads?max_threads=1&max_final_threads=2' -1 -max_final_threads 2 +$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_final_threads' 'http://localhost:8123/query_param_with_url/max_threads?max_threads=1&max_final_threads=2' +max_final_threads 2 +max_threads 1 ``` :::note -In one `predefined_query_handler` only supports one `query` of an insert type. +In one `predefined_query_handler` only one `query` is supported. ::: ### dynamic_query_handler {#dynamic_query_handler} diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index be8cfbdda6c..5f11f1b430b 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -434,16 +434,18 @@ $ curl -v 'http://localhost:8123/predefined_query' ``` xml - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> GET TEST_HEADER_VALUE - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> predefined_query_handler - SELECT value FROM system.settings WHERE name = {name_1:String} - SELECT name, value FROM system.settings WHERE name = {name_2:String} + + SELECT name, value FROM system.settings + WHERE name IN ({name_1:String}, {name_2:String}) + @@ -451,13 +453,13 @@ $ curl -v 'http://localhost:8123/predefined_query' ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/query_param_with_url/1/max_threads/max_final_threads?max_threads=1&max_final_threads=2' -1 -max_final_threads 2 +$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_final_threads' 'http://localhost:8123/query_param_with_url/max_threads?max_threads=1&max_final_threads=2' +max_final_threads 2 +max_threads 1 ``` :::note Предупреждение -В одном `predefined_query_handler` поддерживается только один запрос типа `INSERT`. +В одном `predefined_query_handler` поддерживается только один запрос. ::: ### dynamic_query_handler {#dynamic_query_handler} diff --git a/docs/zh/interfaces/http.md b/docs/zh/interfaces/http.md index 84ca5ed0c47..f55cf41936f 100644 --- a/docs/zh/interfaces/http.md +++ b/docs/zh/interfaces/http.md @@ -427,29 +427,32 @@ $ curl -v 'http://localhost:8123/predefined_query' ``` xml - [^/]+)(/(?P[^/]+))?]]> - GET + [^/]+)]]> + GET TEST_HEADER_VALUE - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> predefined_query_handler - SELECT value FROM system.settings WHERE name = {name_1:String} - SELECT name, value FROM system.settings WHERE name = {name_2:String} + + SELECT name, value FROM system.settings + WHERE name IN ({name_1:String}, {name_2:String}) + + ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/query_param_with_url/1/max_threads/max_final_threads?max_threads=1&max_final_threads=2' -1 -max_final_threads 2 +$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_final_threads' 'http://localhost:8123/query_param_with_url/max_threads?max_threads=1&max_final_threads=2' +max_final_threads 2 +max_threads 1 ``` :::warning -在一个`predefined_query_handler`中，只支持insert类型的一个`查询`。 +在一个`predefined_query_handler`中，只支持的一个`查询`。 ::: ### 动态查询 {#dynamic_query_handler} From f5e9a09d69ea0d1f961464e866c77a73c5c0e82e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Apr 2024 05:20:09 +0200 Subject: [PATCH 1063/1165] Remove support for INSERT WATCH query --- src/Client/ClientBase.cpp | 2 +- src/Interpreters/InterpreterInsertQuery.cpp | 14 +++----------- src/Interpreters/executeQuery.cpp | 11 ----------- src/Parsers/ASTInsertQuery.cpp | 7 +------ src/Parsers/ASTInsertQuery.h | 2 -- src/Parsers/ParserInsertQuery.cpp | 15 +-------------- 6 files changed, 6 insertions(+), 45 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 7a3192d1d9c..8107bd94394 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1964,7 +1964,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin } /// INSERT query for which data transfer is needed (not an INSERT SELECT or input()) is processed separately. - if (insert && (!insert->select || input_function) && !insert->watch && !is_async_insert_with_inlined_data) + if (insert && (!insert->select || input_function) && !is_async_insert_with_inlined_data) { if (input_function && insert->format.empty()) throw Exception(ErrorCodes::INVALID_USAGE_OF_INPUT, "FORMAT must be specified for function input()"); diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index fc58f7b5098..35ff65c2335 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -340,13 +340,10 @@ bool InterpreterInsertQuery::shouldAddSquashingFroStorage(const StoragePtr & tab { auto context_ptr = getContext(); const Settings & settings = context_ptr->getSettingsRef(); - const ASTInsertQuery * query = nullptr; - if (query_ptr) - query = query_ptr->as(); /// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side. /// Client-side bufferization might cause excessive timeouts (especially in case of big blocks). - return !(settings.distributed_foreground_insert && table->isRemote()) && !async_insert && !no_squash && !(query && query->watch); + return !(settings.distributed_foreground_insert && table->isRemote()) && !async_insert && !no_squash; } Chain InterpreterInsertQuery::buildPreSinkChain( @@ -429,7 +426,7 @@ BlockIO InterpreterInsertQuery::execute() std::vector presink_chains; std::vector sink_chains; - if (!distributed_pipeline || query.watch) + if (!distributed_pipeline) { /// Number of streams works like this: /// * For the SELECT, use `max_threads`, or `max_insert_threads`, or whatever @@ -560,11 +557,6 @@ BlockIO InterpreterInsertQuery::execute() } } } - else if (query.watch) - { - InterpreterWatchQuery interpreter_watch{ query.watch, getContext() }; - pipeline = interpreter_watch.buildQueryPipeline(); - } ThreadGroupPtr running_group; if (current_thread) @@ -591,7 +583,7 @@ BlockIO InterpreterInsertQuery::execute() { res.pipeline = std::move(*distributed_pipeline); } - else if (query.select || query.watch) + else if (query.select) { const auto & header = presink_chains.at(0).getInputHeader(); auto actions_dag = ActionsDAG::makeConvertingActions( diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index ea2f69bd2b1..96a9c8d8c8e 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -644,15 +644,6 @@ void logExceptionBeforeStart( } } -static void setQuerySpecificSettings(ASTPtr & ast, ContextMutablePtr context) -{ - if (auto * ast_insert_into = ast->as()) - { - if (ast_insert_into->watch) - context->setSetting("output_format_enable_streaming", 1); - } -} - void validateAnalyzerSettings(ASTPtr ast, bool context_value) { if (ast->as()) @@ -898,8 +889,6 @@ static std::tuple executeQueryImpl( if (auto * insert_query = ast->as()) insert_query->tail = istr; - setQuerySpecificSettings(ast, context); - /// There is an option of probabilistic logging of queries. /// If it is used - do the random sampling and "collapse" the settings. /// It allows to consistently log queries with all the subqueries in distributed query processing diff --git a/src/Parsers/ASTInsertQuery.cpp b/src/Parsers/ASTInsertQuery.cpp index 72a569fe047..8e3458539f3 100644 --- a/src/Parsers/ASTInsertQuery.cpp +++ b/src/Parsers/ASTInsertQuery.cpp @@ -123,13 +123,8 @@ void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & s settings.ostr << delim; select->formatImpl(settings, state, frame); } - else if (watch) - { - settings.ostr << delim; - watch->formatImpl(settings, state, frame); - } - if (!select && !watch) + if (!select) { if (!format.empty()) { diff --git a/src/Parsers/ASTInsertQuery.h b/src/Parsers/ASTInsertQuery.h index b0f444ed755..aeab0f148be 100644 --- a/src/Parsers/ASTInsertQuery.h +++ b/src/Parsers/ASTInsertQuery.h @@ -24,7 +24,6 @@ public: ASTPtr settings_ast; ASTPtr select; - ASTPtr watch; ASTPtr infile; ASTPtr compression; @@ -63,7 +62,6 @@ public: if (partition_by) { res->partition_by = partition_by->clone(); res->children.push_back(res->partition_by); } if (settings_ast) { res->settings_ast = settings_ast->clone(); res->children.push_back(res->settings_ast); } if (select) { res->select = select->clone(); res->children.push_back(res->select); } - if (watch) { res->watch = watch->clone(); res->children.push_back(res->watch); } if (infile) { res->infile = infile->clone(); res->children.push_back(res->infile); } if (compression) { res->compression = compression->clone(); res->children.push_back(res->compression); } diff --git a/src/Parsers/ParserInsertQuery.cpp b/src/Parsers/ParserInsertQuery.cpp index d1171dd4815..9373e6a1c93 100644 --- a/src/Parsers/ParserInsertQuery.cpp +++ b/src/Parsers/ParserInsertQuery.cpp @@ -36,7 +36,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_format(Keyword::FORMAT); ParserKeyword s_settings(Keyword::SETTINGS); ParserKeyword s_select(Keyword::SELECT); - ParserKeyword s_watch(Keyword::WATCH); ParserKeyword s_partition_by(Keyword::PARTITION_BY); ParserKeyword s_with(Keyword::WITH); ParserToken s_lparen(TokenType::OpeningRoundBracket); @@ -56,7 +55,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr columns; ASTPtr format; ASTPtr select; - ASTPtr watch; ASTPtr table_function; ASTPtr settings_ast; ASTPtr partition_by_expr; @@ -143,7 +141,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) String format_str; Pos before_values = pos; - /// VALUES or FORMAT or SELECT or WITH or WATCH. + /// VALUES or FORMAT or SELECT or WITH. /// After FROM INFILE we expect FORMAT, SELECT, WITH or nothing. if (!infile && s_values.ignore(pos, expected)) { @@ -175,14 +173,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) tryGetIdentifierNameInto(format, format_str); } - else if (!infile && s_watch.ignore(pos, expected)) - { - /// If WATCH is defined, return to position before WATCH and parse - /// rest of query as WATCH query. - pos = before_values; - ParserWatchQuery watch_p; - watch_p.parse(pos, watch, expected); - } else if (!infile) { /// If all previous conditions were false and it's not FROM INFILE, query is incorrect @@ -286,7 +276,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) query->columns = columns; query->format = std::move(format_str); query->select = select; - query->watch = watch; query->settings_ast = settings_ast; query->data = data != end ? data : nullptr; query->end = end; @@ -295,8 +284,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) query->children.push_back(columns); if (select) query->children.push_back(select); - if (watch) - query->children.push_back(watch); if (settings_ast) query->children.push_back(settings_ast); From fb31ad1736aa6bfc758f40a86dcac5afbc07a01b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 8 Apr 2024 06:57:56 +0000 Subject: [PATCH 1064/1165] Fix another test --- src/Core/SettingsChangesHistory.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index f43ca154d56..7fa12780c8c 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -87,6 +87,7 @@ static std::map sett { {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"}, + {"query_cache_system_table_handling", QueryCacheSystemTableHandling::Save, QueryCacheSystemTableHandling::Throw, "The query cache no longer caches results of queries against system tables"}, }}, {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, From 057893c3107a3e14f4ec2d9c54a2664129125de4 Mon Sep 17 00:00:00 2001 From: peter279k Date: Mon, 8 Apr 2024 14:32:46 +0800 Subject: [PATCH 1065/1165] Add checksum to validate the downloaded archive --- docs/en/getting-started/example-datasets/menus.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/en/getting-started/example-datasets/menus.md b/docs/en/getting-started/example-datasets/menus.md index 32fe62865d4..5a35c1d45bc 100644 --- a/docs/en/getting-started/example-datasets/menus.md +++ b/docs/en/getting-started/example-datasets/menus.md @@ -18,6 +18,9 @@ Run the command: ```bash wget https://s3.amazonaws.com/menusdata.nypl.org/gzips/2021_08_01_07_01_17_data.tgz +# Option: Validate the checksum +md5sum 2021_08_01_07_01_17_data.tgz +# Checksum should be equal to: db6126724de939a5481e3160a2d67d15 ``` Replace the link to the up to date link from http://menus.nypl.org/data if needed. From b369291d47acb9e2e37cd91ea7063d13087259fa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 1 Apr 2024 01:09:52 +0200 Subject: [PATCH 1066/1165] Rich syntax highlighting in the client --- src/Client/ClientBaseHelpers.cpp | 123 ++++++++++------------- src/Parsers/ASTOrderByElement.cpp | 1 - src/Parsers/CommonParsers.h | 2 + src/Parsers/ExpressionElementParsers.cpp | 2 +- src/Parsers/ExpressionElementParsers.h | 14 ++- src/Parsers/ExpressionListParsers.cpp | 91 ++++++++++------- src/Parsers/IParser.cpp | 23 +++++ src/Parsers/IParser.h | 37 +++++++ src/Parsers/IParserBase.cpp | 15 ++- 9 files changed, 198 insertions(+), 110 deletions(-) diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index b0862696295..22f6c8912b1 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -1,11 +1,11 @@ #include "ClientBaseHelpers.h" - #include #include -#include +#include #include + namespace DB { @@ -96,77 +96,64 @@ void highlight(const String & query, std::vector & colors { using namespace replxx; - static const std::unordered_map token_to_color - = {{TokenType::Whitespace, Replxx::Color::DEFAULT}, - {TokenType::Comment, Replxx::Color::GRAY}, - {TokenType::BareWord, Replxx::Color::DEFAULT}, - {TokenType::Number, Replxx::Color::GREEN}, - {TokenType::StringLiteral, Replxx::Color::CYAN}, - {TokenType::QuotedIdentifier, Replxx::Color::MAGENTA}, - {TokenType::OpeningRoundBracket, Replxx::Color::BROWN}, - {TokenType::ClosingRoundBracket, Replxx::Color::BROWN}, - {TokenType::OpeningSquareBracket, Replxx::Color::BROWN}, - {TokenType::ClosingSquareBracket, Replxx::Color::BROWN}, - {TokenType::DoubleColon, Replxx::Color::BROWN}, - {TokenType::OpeningCurlyBrace, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::ClosingCurlyBrace, replxx::color::bold(Replxx::Color::DEFAULT)}, + if (colors.empty()) + return; - {TokenType::Comma, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Semicolon, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::VerticalDelimiter, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Dot, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Asterisk, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::HereDoc, Replxx::Color::CYAN}, - {TokenType::Plus, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Minus, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Slash, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Percent, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Arrow, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::QuestionMark, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Colon, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Equals, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::NotEquals, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Less, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Greater, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::LessOrEquals, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::GreaterOrEquals, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Spaceship, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::Concatenation, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::At, replxx::color::bold(Replxx::Color::DEFAULT)}, - {TokenType::DoubleAt, Replxx::Color::MAGENTA}, - - {TokenType::EndOfStream, Replxx::Color::DEFAULT}, - - {TokenType::Error, Replxx::Color::RED}, - {TokenType::ErrorMultilineCommentIsNotClosed, Replxx::Color::RED}, - {TokenType::ErrorSingleQuoteIsNotClosed, Replxx::Color::RED}, - {TokenType::ErrorDoubleQuoteIsNotClosed, Replxx::Color::RED}, - {TokenType::ErrorSinglePipeMark, Replxx::Color::RED}, - {TokenType::ErrorWrongNumber, Replxx::Color::RED}, - {TokenType::ErrorMaxQuerySizeExceeded, Replxx::Color::RED}}; - - const Replxx::Color unknown_token_color = Replxx::Color::RED; - - Lexer lexer(query.data(), query.data() + query.size()); - size_t pos = 0; - - for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken()) + static const std::unordered_map type_to_color = { - if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter) - ReplxxLineReader::setLastIsDelimiter(true); - else if (token.type != TokenType::Whitespace) - ReplxxLineReader::setLastIsDelimiter(false); + {Highlight::keyword, replxx::color::bold(Replxx::Color::DEFAULT)}, + {Highlight::identifier, Replxx::Color::CYAN}, + {Highlight::function, Replxx::Color::BROWN}, + {Highlight::alias, Replxx::Color::MAGENTA}, + {Highlight::substitution, Replxx::Color::MAGENTA}, + {Highlight::number, Replxx::Color::BRIGHTGREEN}, + {Highlight::string, Replxx::Color::GREEN}, + }; - size_t utf8_len = UTF8::countCodePoints(reinterpret_cast(token.begin), token.size()); - for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index) + const char * begin = query.data(); + const char * end = begin + query.size(); + Tokens tokens(begin, end, 1000, true); + IParser::Pos token_iterator(tokens, static_cast(1000), static_cast(10000)); + Expected expected; + ParserQuery parser(end); + ASTPtr ast; + bool parse_res = false; + + try + { + parse_res = parser.parse(token_iterator, ast, expected); + } + catch (...) + { + return; + } + + size_t pos = 0; + const char * prev = begin; + for (const auto & range : expected.highlights) + { + auto it = type_to_color.find(range.highlight); + if (it != type_to_color.end()) { - if (token_to_color.find(token.type) != token_to_color.end()) - colors[pos + code_point_index] = token_to_color.at(token.type); - else - colors[pos + code_point_index] = unknown_token_color; - } + pos += UTF8::countCodePoints(reinterpret_cast(prev), range.begin - prev); + size_t utf8_len = UTF8::countCodePoints(reinterpret_cast(range.begin), range.end - range.begin); - pos += utf8_len; + for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index) + colors[pos + code_point_index] = it->second; + + pos += utf8_len; + prev = range.end; + } + } + + if (!parse_res) + { + pos += UTF8::countCodePoints(reinterpret_cast(prev), expected.max_parsed_pos - prev); + + if (pos >= colors.size()) + pos = colors.size() - 1; + + colors[pos] = Replxx::Color::BRIGHTRED; } } #endif diff --git a/src/Parsers/ASTOrderByElement.cpp b/src/Parsers/ASTOrderByElement.cpp index be0416359a1..09193a8b5e1 100644 --- a/src/Parsers/ASTOrderByElement.cpp +++ b/src/Parsers/ASTOrderByElement.cpp @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index 49964b5c728..2277e348b0f 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -601,6 +601,8 @@ public: constexpr const char * getName() const override { return s.data(); } + Highlight highlight() const override { return Highlight::keyword; } + protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 2c8ab65d1fc..d5a67c09f9f 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -278,7 +278,7 @@ bool ParserTableAsStringLiteralIdentifier::parseImpl(Pos & pos, ASTPtr & node, E bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTPtr id_list; - if (!ParserList(std::make_unique(allow_query_parameter), std::make_unique(TokenType::Dot), false) + if (!ParserList(std::make_unique(allow_query_parameter, highlight_type), std::make_unique(TokenType::Dot), false) .parse(pos, id_list, expected)) return false; diff --git a/src/Parsers/ExpressionElementParsers.h b/src/Parsers/ExpressionElementParsers.h index b29f5cc4251..61e35690938 100644 --- a/src/Parsers/ExpressionElementParsers.h +++ b/src/Parsers/ExpressionElementParsers.h @@ -25,12 +25,15 @@ protected: class ParserIdentifier : public IParserBase { public: - explicit ParserIdentifier(bool allow_query_parameter_ = false) : allow_query_parameter(allow_query_parameter_) {} + explicit ParserIdentifier(bool allow_query_parameter_ = false, Highlight highlight_type_ = Highlight::identifier) + : allow_query_parameter(allow_query_parameter_), highlight_type(highlight_type_) {} + Highlight highlight() const override { return highlight_type; } protected: const char * getName() const override { return "identifier"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; bool allow_query_parameter; + Highlight highlight_type; }; @@ -53,8 +56,8 @@ protected: class ParserCompoundIdentifier : public IParserBase { public: - explicit ParserCompoundIdentifier(bool table_name_with_optional_uuid_ = false, bool allow_query_parameter_ = false) - : table_name_with_optional_uuid(table_name_with_optional_uuid_), allow_query_parameter(allow_query_parameter_) + explicit ParserCompoundIdentifier(bool table_name_with_optional_uuid_ = false, bool allow_query_parameter_ = false, Highlight highlight_type_ = Highlight::identifier) + : table_name_with_optional_uuid(table_name_with_optional_uuid_), allow_query_parameter(allow_query_parameter_), highlight_type(highlight_type_) { } @@ -63,6 +66,7 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; bool table_name_with_optional_uuid; bool allow_query_parameter; + Highlight highlight_type; }; /** *, t.*, db.table.*, COLUMNS('') APPLY(...) or EXCEPT(...) or REPLACE(...) @@ -253,6 +257,7 @@ class ParserNumber : public IParserBase protected: const char * getName() const override { return "number"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + Highlight highlight() const override { return Highlight::number; } }; /** Unsigned integer, used in right hand side of tuple access operator (x.1). @@ -273,6 +278,7 @@ class ParserStringLiteral : public IParserBase protected: const char * getName() const override { return "string literal"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + Highlight highlight() const override { return Highlight::string; } }; @@ -363,6 +369,7 @@ private: const char * getName() const override { return "alias"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + Highlight highlight() const override { return Highlight::alias; } }; @@ -385,6 +392,7 @@ class ParserSubstitution : public IParserBase protected: const char * getName() const override { return "substitution"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + Highlight highlight() const override { return Highlight::substitution; } }; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 05691529f43..276b4e82074 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -441,6 +441,21 @@ bool ParserKeyValuePairsList::parseImpl(Pos & pos, ASTPtr & node, Expected & exp return parser.parse(pos, node, expected); } +namespace +{ + /// This wrapper is needed to highlight function names differently. + class ParserFunctionName : public IParserBase + { + protected: + const char * getName() const override { return "function name"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override + { + ParserCompoundIdentifier parser(false, true, Highlight::function); + return parser.parse(pos, node, expected); + } + }; +} + enum class Action { @@ -809,6 +824,7 @@ struct ParserExpressionImpl static const Operator finish_between_operator; + ParserFunctionName function_name_parser; ParserCompoundIdentifier identifier_parser{false, true}; ParserNumber number_parser; ParserAsterisk asterisk_parser; @@ -2359,7 +2375,7 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTPtr identifier; - if (ParserCompoundIdentifier(false,true).parse(pos, identifier, expected) + if (ParserFunctionName().parse(pos, identifier, expected) && ParserToken(TokenType::OpeningRoundBracket).ignore(pos, expected)) { auto start = getFunctionLayer(identifier, is_table_function, allow_function_parameters); @@ -2497,7 +2513,7 @@ Action ParserExpressionImpl::tryParseOperand(Layers & layers, IParser::Pos & pos { if (typeid_cast(layers.back().get()) || typeid_cast(layers.back().get())) { - if (identifier_parser.parse(pos, tmp, expected) + if (function_name_parser.parse(pos, tmp, expected) && ParserToken(TokenType::OpeningRoundBracket).ignore(pos, expected)) { layers.push_back(getFunctionLayer(tmp, layers.front()->is_table_function)); @@ -2629,50 +2645,53 @@ Action ParserExpressionImpl::tryParseOperand(Layers & layers, IParser::Pos & pos { layers.back()->pushOperand(std::move(tmp)); } - else if (identifier_parser.parse(pos, tmp, expected)) + else { - if (pos->type == TokenType::OpeningRoundBracket) + old_pos = pos; + if (function_name_parser.parse(pos, tmp, expected) && pos->type == TokenType::OpeningRoundBracket) { ++pos; layers.push_back(getFunctionLayer(tmp, layers.front()->is_table_function)); return Action::OPERAND; } + pos = old_pos; + + if (identifier_parser.parse(pos, tmp, expected)) + { + layers.back()->pushOperand(std::move(tmp)); + } + else if (substitution_parser.parse(pos, tmp, expected)) + { + layers.back()->pushOperand(std::move(tmp)); + } + else if (pos->type == TokenType::OpeningRoundBracket) + { + + if (subquery_parser.parse(pos, tmp, expected)) + { + layers.back()->pushOperand(std::move(tmp)); + return Action::OPERATOR; + } + + ++pos; + layers.push_back(std::make_unique()); + return Action::OPERAND; + } + else if (pos->type == TokenType::OpeningSquareBracket) + { + ++pos; + layers.push_back(std::make_unique()); + return Action::OPERAND; + } + else if (mysql_global_variable_parser.parse(pos, tmp, expected)) + { + layers.back()->pushOperand(std::move(tmp)); + } else { - layers.back()->pushOperand(std::move(tmp)); + return Action::NONE; } } - else if (substitution_parser.parse(pos, tmp, expected)) - { - layers.back()->pushOperand(std::move(tmp)); - } - else if (pos->type == TokenType::OpeningRoundBracket) - { - - if (subquery_parser.parse(pos, tmp, expected)) - { - layers.back()->pushOperand(std::move(tmp)); - return Action::OPERATOR; - } - - ++pos; - layers.push_back(std::make_unique()); - return Action::OPERAND; - } - else if (pos->type == TokenType::OpeningSquareBracket) - { - ++pos; - layers.push_back(std::make_unique()); - return Action::OPERAND; - } - else if (mysql_global_variable_parser.parse(pos, tmp, expected)) - { - layers.back()->pushOperand(std::move(tmp)); - } - else - { - return Action::NONE; - } return Action::OPERATOR; } diff --git a/src/Parsers/IParser.cpp b/src/Parsers/IParser.cpp index 41981a4bb8a..eb4ddfa01d2 100644 --- a/src/Parsers/IParser.cpp +++ b/src/Parsers/IParser.cpp @@ -9,6 +9,7 @@ namespace ErrorCodes extern const int TOO_SLOW_PARSING; } + IParser::Pos & IParser::Pos::operator=(const IParser::Pos & rhs) { depth = rhs.depth; @@ -32,4 +33,26 @@ IParser::Pos & IParser::Pos::operator=(const IParser::Pos & rhs) return *this; } + +template +static bool intersects(T a_begin, T a_end, T b_begin, T b_end) +{ + return (a_begin <= b_begin && b_begin < a_end) + || (b_begin <= a_begin && a_begin < b_end); +} + + +void Expected::highlight(HighlightedRange range) +{ + auto it = highlights.lower_bound(range); + while (it != highlights.end() && range.begin < it->end) + { + if (intersects(range.begin, range.end, it->begin, it->end)) + it = highlights.erase(it); + else + ++it; + } + highlights.insert(range); +} + } diff --git a/src/Parsers/IParser.h b/src/Parsers/IParser.h index 291f8ee7d44..d79bc0fb999 100644 --- a/src/Parsers/IParser.h +++ b/src/Parsers/IParser.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -21,6 +22,30 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +enum class Highlight +{ + none = 0, + keyword, + identifier, + function, + alias, + substitution, + number, + string, +}; + +struct HighlightedRange +{ + const char * begin; + const char * end; + Highlight highlight; + + auto operator<=>(const HighlightedRange & other) const + { + return begin <=> other.begin; + } +}; + /** Collects variants, how parser could proceed further at rightmost position. */ @@ -29,6 +54,8 @@ struct Expected absl::InlinedVector variants; const char * max_parsed_pos = nullptr; + std::set highlights; + /// 'description' should be statically allocated string. ALWAYS_INLINE void add(const char * current_pos, const char * description) { @@ -48,6 +75,8 @@ struct Expected { add(it->begin, description); } + + void highlight(HighlightedRange range); }; @@ -158,6 +187,14 @@ public: return parse(pos, node, expected); } + /** If the parsed fragment should be highlighted in the query editor, + * which type of highlighting to use? + */ + virtual Highlight highlight() const + { + return Highlight::none; + } + virtual ~IParser() = default; }; diff --git a/src/Parsers/IParserBase.cpp b/src/Parsers/IParserBase.cpp index 0241250926d..1293dd7dacb 100644 --- a/src/Parsers/IParserBase.cpp +++ b/src/Parsers/IParserBase.cpp @@ -10,8 +10,21 @@ bool IParserBase::parse(Pos & pos, ASTPtr & node, Expected & expected) return wrapParseImpl(pos, IncreaseDepthTag{}, [&] { + const char * begin = pos->begin; bool res = parseImpl(pos, node, expected); - if (!res) + if (res) + { + Highlight type = highlight(); + if (type != Highlight::none) + { + HighlightedRange range; + range.begin = begin; + range.end = pos->begin; + range.highlight = type; + expected.highlight(range); + } + } + else node = nullptr; return res; }); From b9406f79d168f6a18881bbabc89826de36a155a1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Apr 2024 23:10:34 +0200 Subject: [PATCH 1067/1165] Better highlighting --- src/Client/ClientBaseHelpers.cpp | 4 +++- src/Parsers/IParserBase.cpp | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 22f6c8912b1..ce4ee9a0559 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -146,7 +146,9 @@ void highlight(const String & query, std::vector & colors } } - if (!parse_res) + Token last_token = token_iterator.max(); + + if (!parse_res || last_token.isError() || (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon)) { pos += UTF8::countCodePoints(reinterpret_cast(prev), expected.max_parsed_pos - prev); diff --git a/src/Parsers/IParserBase.cpp b/src/Parsers/IParserBase.cpp index 1293dd7dacb..9d39056a8f1 100644 --- a/src/Parsers/IParserBase.cpp +++ b/src/Parsers/IParserBase.cpp @@ -15,12 +15,16 @@ bool IParserBase::parse(Pos & pos, ASTPtr & node, Expected & expected) if (res) { Highlight type = highlight(); - if (type != Highlight::none) + if (pos->begin > begin && type != Highlight::none) { + Pos prev_token = pos; + --prev_token; + HighlightedRange range; range.begin = begin; - range.end = pos->begin; + range.end = prev_token->end; range.highlight = type; + expected.highlight(range); } } From c8d6dc47c3f66a1bb6bae2c6d0f2f13d64173fff Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Apr 2024 23:18:18 +0200 Subject: [PATCH 1068/1165] Fix test --- src/Client/ClientBaseHelpers.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index ce4ee9a0559..01818589499 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -125,6 +125,8 @@ void highlight(const String & query, std::vector & colors } catch (...) { + /// Skip highlighting in the case of exceptions during parsing. + /// It is ok to ignore unknown exceptions here. return; } @@ -157,6 +159,11 @@ void highlight(const String & query, std::vector & colors colors[pos] = Replxx::Color::BRIGHTRED; } + + if (last_token.type == TokenType::Semicolon || last_token.type == TokenType::VerticalDelimiter) + ReplxxLineReader::setLastIsDelimiter(true); + else if (last_token.type != TokenType::Whitespace) + ReplxxLineReader::setLastIsDelimiter(false); } #endif From fb664fa99f400d9a9e5ef861fb6a91b62a5d35a5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Apr 2024 01:47:47 +0200 Subject: [PATCH 1069/1165] Slightly better --- src/Client/ClientBaseHelpers.cpp | 13 +++++++-- src/Parsers/parseQuery.cpp | 49 ++++++++++++++++++-------------- src/Parsers/parseQuery.h | 5 ++++ 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 01818589499..c08e015e1c5 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -149,8 +150,11 @@ void highlight(const String & query, std::vector & colors } Token last_token = token_iterator.max(); + /// Raw data in INSERT queries, which is not necessarily tokenized. + const char * insert_data = ast ? getInsertData(ast) : nullptr; - if (!parse_res || last_token.isError() || (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon)) + if ((!parse_res || last_token.isError() || (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon)) + && !(insert_data && expected.max_parsed_pos >= insert_data)) { pos += UTF8::countCodePoints(reinterpret_cast(prev), expected.max_parsed_pos - prev); @@ -160,10 +164,15 @@ void highlight(const String & query, std::vector & colors colors[pos] = Replxx::Color::BRIGHTRED; } - if (last_token.type == TokenType::Semicolon || last_token.type == TokenType::VerticalDelimiter) + if (last_token.type == TokenType::Semicolon || last_token.type == TokenType::VerticalDelimiter + || query.ends_with(';') || query.ends_with("\\G")) /// This is for raw data in INSERT queries, which is not necessarily tokenized. + { ReplxxLineReader::setLastIsDelimiter(true); + } else if (last_token.type != TokenType::Whitespace) + { ReplxxLineReader::setLastIsDelimiter(false); + } } #endif diff --git a/src/Parsers/parseQuery.cpp b/src/Parsers/parseQuery.cpp index 51878efa706..c6727a36995 100644 --- a/src/Parsers/parseQuery.cpp +++ b/src/Parsers/parseQuery.cpp @@ -226,6 +226,29 @@ std::string getUnmatchedParenthesesErrorMessage( } +const char * getInsertData(const ASTPtr & ast) +{ + /// Either it is INSERT or EXPLAIN INSERT. + + ASTInsertQuery * insert = nullptr; + if (auto * explain = ast->as()) + { + if (auto explained_query = explain->getExplainedQuery()) + { + insert = explained_query->as(); + } + } + else + { + insert = ast->as(); + } + + if (insert) + return insert->data; + return nullptr; +} + + ASTPtr tryParseQuery( IParser & parser, const char * & _out_query_end, /* also query begin as input parameter */ @@ -270,29 +293,11 @@ ASTPtr tryParseQuery( if (res && max_parser_depth) res->checkDepth(max_parser_depth); - ASTInsertQuery * insert = nullptr; - if (parse_res) - { - if (auto * explain = res->as()) - { - if (auto explained_query = explain->getExplainedQuery()) - { - insert = explained_query->as(); - } - } - else - { - insert = res->as(); - } - } - - // If parsed query ends at data for insertion. Data for insertion could be - // in any format and not necessary be lexical correct, so we can't perform - // most of the checks. - if (insert && insert->data) - { + /// If parsed query ends at data for insertion. Data for insertion could be + /// in any format and not necessary be lexical correct, so we can't perform + /// most of the checks. + if (res && getInsertData(res)) return res; - } // More granular checks for queries other than INSERT w/inline data. /// Lexical error diff --git a/src/Parsers/parseQuery.h b/src/Parsers/parseQuery.h index 93c1a465267..564415d0b85 100644 --- a/src/Parsers/parseQuery.h +++ b/src/Parsers/parseQuery.h @@ -71,4 +71,9 @@ std::pair splitMultipartQuery( size_t max_parser_backtracks, bool allow_settings_after_format_in_insert); +/** If the query contains raw data part, such as INSERT ... FORMAT ..., return a pointer to it. + * The SQL parser stops at the raw data part, which is parsed by a separate parser. + */ +const char * getInsertData(const ASTPtr & ast); + } From 0ff26d2d777b2bd861fc93d8a337993eb292a260 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Apr 2024 23:23:36 +0200 Subject: [PATCH 1070/1165] A little better --- src/Client/ClientBaseHelpers.cpp | 6 +++--- src/Parsers/ExpressionElementParsers.cpp | 2 +- src/Parsers/ExpressionElementParsers.h | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index c08e015e1c5..4dcd025d9fc 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -105,9 +105,9 @@ void highlight(const String & query, std::vector & colors {Highlight::keyword, replxx::color::bold(Replxx::Color::DEFAULT)}, {Highlight::identifier, Replxx::Color::CYAN}, {Highlight::function, Replxx::Color::BROWN}, - {Highlight::alias, Replxx::Color::MAGENTA}, + {Highlight::alias, replxx::color::rgb666(0, 4, 4)}, {Highlight::substitution, Replxx::Color::MAGENTA}, - {Highlight::number, Replxx::Color::BRIGHTGREEN}, + {Highlight::number, replxx::color::rgb666(0, 4, 0)}, {Highlight::string, Replxx::Color::GREEN}, }; @@ -161,7 +161,7 @@ void highlight(const String & query, std::vector & colors if (pos >= colors.size()) pos = colors.size() - 1; - colors[pos] = Replxx::Color::BRIGHTRED; + colors[pos] = replxx::color::bg(replxx::color::rgb666(5, 3, 3)); } if (last_token.type == TokenType::Semicolon || last_token.type == TokenType::VerticalDelimiter diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index d5a67c09f9f..dce0bc62d5b 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1491,7 +1491,7 @@ const char * ParserAlias::restricted_keywords[] = bool ParserAlias::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_as(Keyword::AS); - ParserIdentifier id_p; + ParserIdentifier id_p(false, Highlight::alias); bool has_as_word = s_as.ignore(pos, expected); if (!allow_alias_without_as_keyword && !has_as_word) diff --git a/src/Parsers/ExpressionElementParsers.h b/src/Parsers/ExpressionElementParsers.h index 61e35690938..6dbb75450ed 100644 --- a/src/Parsers/ExpressionElementParsers.h +++ b/src/Parsers/ExpressionElementParsers.h @@ -369,7 +369,6 @@ private: const char * getName() const override { return "alias"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; - Highlight highlight() const override { return Highlight::alias; } }; From 537f045c1ce258757eda7fce5c639461e9449114 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Apr 2024 04:19:34 +0200 Subject: [PATCH 1071/1165] Miscellaneous --- src/Client/ClientBase.cpp | 8 ++++---- src/Client/ClientBaseHelpers.cpp | 2 +- src/Parsers/parseDatabaseAndTableName.cpp | 15 --------------- src/Parsers/parseQuery.cpp | 21 +++++++++++---------- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 7a3192d1d9c..248448f36c6 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2056,7 +2056,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText( return MultiQueryProcessingStage::QUERIES_END; // Remove leading empty newlines and other whitespace, because they - // are annoying to filter in query log. This is mostly relevant for + // are annoying to filter in the query log. This is mostly relevant for // the tests. while (this_query_begin < all_queries_end && isWhitespaceASCII(*this_query_begin)) ++this_query_begin; @@ -2086,7 +2086,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText( { parsed_query = parseQuery(this_query_end, all_queries_end, true); } - catch (Exception & e) + catch (const Exception & e) { current_exception.reset(e.clone()); return MultiQueryProcessingStage::PARSING_EXCEPTION; @@ -2111,9 +2111,9 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText( // INSERT queries may have the inserted data in the query text // that follow the query itself, e.g. "insert into t format CSV 1;2". // They need special handling. First of all, here we find where the - // inserted data ends. In multy-query mode, it is delimited by a + // inserted data ends. In multi-query mode, it is delimited by a // newline. - // The VALUES format needs even more handling -- we also allow the + // The VALUES format needs even more handling - we also allow the // data to be delimited by semicolon. This case is handled later by // the format parser itself. // We can't do multiline INSERTs with inline data, because most diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 4dcd025d9fc..da164bd1226 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -161,7 +161,7 @@ void highlight(const String & query, std::vector & colors if (pos >= colors.size()) pos = colors.size() - 1; - colors[pos] = replxx::color::bg(replxx::color::rgb666(5, 3, 3)); + colors[pos] = Replxx::Color::BRIGHTRED; } if (last_token.type == TokenType::Semicolon || last_token.type == TokenType::VerticalDelimiter diff --git a/src/Parsers/parseDatabaseAndTableName.cpp b/src/Parsers/parseDatabaseAndTableName.cpp index 81660bc4600..eaf020e445b 100644 --- a/src/Parsers/parseDatabaseAndTableName.cpp +++ b/src/Parsers/parseDatabaseAndTableName.cpp @@ -60,21 +60,6 @@ bool parseDatabaseAndTableAsAST(IParser::Pos & pos, Expected & expected, ASTPtr } -bool parseDatabase(IParser::Pos & pos, Expected & expected, String & database_str) -{ - ParserToken s_dot(TokenType::Dot); - ParserIdentifier identifier_parser; - - ASTPtr database; - database_str = ""; - - if (!identifier_parser.parse(pos, database, expected)) - return false; - - tryGetIdentifierNameInto(database, database_str); - return true; -} - bool parseDatabaseAsAST(IParser::Pos & pos, Expected & expected, ASTPtr & database) { ParserIdentifier identifier_parser(/* allow_query_parameter */true); diff --git a/src/Parsers/parseQuery.cpp b/src/Parsers/parseQuery.cpp index c6727a36995..2a6abc23406 100644 --- a/src/Parsers/parseQuery.cpp +++ b/src/Parsers/parseQuery.cpp @@ -226,24 +226,27 @@ std::string getUnmatchedParenthesesErrorMessage( } -const char * getInsertData(const ASTPtr & ast) +static ASTInsertQuery * getInsertAST(const ASTPtr & ast) { /// Either it is INSERT or EXPLAIN INSERT. - - ASTInsertQuery * insert = nullptr; if (auto * explain = ast->as()) { if (auto explained_query = explain->getExplainedQuery()) { - insert = explained_query->as(); + return explained_query->as(); } } else { - insert = ast->as(); + return ast->as(); } - if (insert) + return nullptr; +} + +const char * getInsertData(const ASTPtr & ast) +{ + if (const ASTInsertQuery * insert = getInsertAST(ast)) return insert->data; return nullptr; } @@ -439,11 +442,9 @@ std::pair splitMultipartQuery( ast = parseQueryAndMovePosition(parser, pos, end, "", true, max_query_size, max_parser_depth, max_parser_backtracks); - auto * insert = ast->as(); - - if (insert && insert->data) + if (ASTInsertQuery * insert = getInsertAST(ast)) { - /// Data for INSERT is broken on new line + /// Data for INSERT is broken on the new line pos = insert->data; while (*pos && *pos != '\n') ++pos; From 426104a2d61dea08ccc7c64052cb5c1a0de66d9d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Apr 2024 04:30:22 +0200 Subject: [PATCH 1072/1165] Fix tests --- ..._autocomplete_word_break_characters.expect | 2 +- .../01676_clickhouse_client_autocomplete.sh | 2 +- .../01702_system_query_log.reference | 20 +++++++++---------- ...160_client_autocomplete_parse_query.expect | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect index 44f3ba9681a..ffd3e742cec 100755 --- a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect +++ b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect @@ -20,7 +20,7 @@ expect_after { -i $any_spawn_id timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file --highlight=0" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh index ebd6490077e..f04ffdae229 100755 --- a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh +++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh @@ -43,7 +43,7 @@ expect_after { -i \$any_spawn_id timeout { exit 1 } } -spawn bash -c "$*" +spawn bash -c "$* --highlight 0" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/01702_system_query_log.reference b/tests/queries/0_stateless/01702_system_query_log.reference index c653021aa5a..5498b5377ba 100644 --- a/tests/queries/0_stateless/01702_system_query_log.reference +++ b/tests/queries/0_stateless/01702_system_query_log.reference @@ -43,16 +43,16 @@ Alter ALTER TABLE sqllt.table UPDATE i = i + 1 WHERE 1; Alter ALTER TABLE sqllt.table DELETE WHERE i > 65535; Select -- not done, seems to hard, so I\'ve skipped queries of ALTER-X, where X is:\n-- PARTITION\n-- ORDER BY\n-- SAMPLE BY\n-- INDEX\n-- CONSTRAINT\n-- TTL\n-- USER\n-- QUOTA\n-- ROLE\n-- ROW POLICY\n-- SETTINGS PROFILE\n\nSELECT \'SYSTEM queries\'; System SYSTEM FLUSH LOGS; -System SYSTEM STOP MERGES sqllt.table -System SYSTEM START MERGES sqllt.table -System SYSTEM STOP TTL MERGES sqllt.table -System SYSTEM START TTL MERGES sqllt.table -System SYSTEM STOP MOVES sqllt.table -System SYSTEM START MOVES sqllt.table -System SYSTEM STOP FETCHES sqllt.table -System SYSTEM START FETCHES sqllt.table -System SYSTEM STOP REPLICATED SENDS sqllt.table -System SYSTEM START REPLICATED SENDS sqllt.table +System SYSTEM STOP MERGES sqllt.table; +System SYSTEM START MERGES sqllt.table; +System SYSTEM STOP TTL MERGES sqllt.table; +System SYSTEM START TTL MERGES sqllt.table; +System SYSTEM STOP MOVES sqllt.table; +System SYSTEM START MOVES sqllt.table; +System SYSTEM STOP FETCHES sqllt.table; +System SYSTEM START FETCHES sqllt.table; +System SYSTEM STOP REPLICATED SENDS sqllt.table; +System SYSTEM START REPLICATED SENDS sqllt.table; Select -- SYSTEM RELOAD DICTIONARY sqllt.dictionary; -- temporary out of order: Code: 210, Connection refused (localhost:9001) (version 21.3.1.1)\n-- DROP REPLICA\n-- haha, no\n-- SYSTEM KILL;\n-- SYSTEM SHUTDOWN;\n\n-- Since we don\'t really care about the actual output, suppress it with `FORMAT Null`.\nSELECT \'SHOW queries\'; Show SHOW CREATE TABLE sqllt.table FORMAT Null; Show SHOW CREATE DICTIONARY sqllt.dictionary FORMAT Null; diff --git a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect index 2d404b005c7..30d725e6a2a 100755 --- a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect +++ b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect @@ -21,7 +21,7 @@ expect_after { -i $any_spawn_id timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file --highlight=0" expect ":) " # Make a query From bd3f32dc0680c8581b68e5200ff9244a2cdde3a0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Apr 2024 05:34:41 +0200 Subject: [PATCH 1073/1165] Fix a test --- src/Parsers/ParserInsertQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Parsers/ParserInsertQuery.cpp b/src/Parsers/ParserInsertQuery.cpp index d1171dd4815..3e691b13ef2 100644 --- a/src/Parsers/ParserInsertQuery.cpp +++ b/src/Parsers/ParserInsertQuery.cpp @@ -41,7 +41,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_with(Keyword::WITH); ParserToken s_lparen(TokenType::OpeningRoundBracket); ParserToken s_rparen(TokenType::ClosingRoundBracket); - ParserToken s_semicolon(TokenType::Semicolon); ParserIdentifier name_p(true); ParserList columns_p(std::make_unique(), std::make_unique(TokenType::Comma), false); ParserFunction table_function_p{false}; @@ -149,8 +148,9 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { /// If VALUES is defined in query, everything except setting will be parsed as data, /// and if values followed by semicolon, the data should be null. - if (!s_semicolon.checkWithoutMoving(pos, expected)) + if (pos->type != TokenType::Semicolon) data = pos->begin; + format_str = "Values"; } else if (s_format.ignore(pos, expected)) From 28272873faa90c0b87d5db3f53df91b40ef71077 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Apr 2024 05:49:02 +0200 Subject: [PATCH 1074/1165] Fix test --- ...01565_query_loop_after_client_error.expect | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/queries/0_stateless/01565_query_loop_after_client_error.expect b/tests/queries/0_stateless/01565_query_loop_after_client_error.expect index ac69c18ce39..6253840c63c 100755 --- a/tests/queries/0_stateless/01565_query_loop_after_client_error.expect +++ b/tests/queries/0_stateless/01565_query_loop_after_client_error.expect @@ -24,30 +24,21 @@ expect_after { -i $any_spawn_id timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn --history_file=$history_file" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn --history_file=$history_file --highlight 0" expect "\n:) " -send -- "DROP TABLE IF EXISTS t01565;\n" -# NOTE: this is important for -mn mode, you should send "\r" only after reading echoed command -expect "\r\n" -send -- "\r" +send -- "DROP TABLE IF EXISTS t01565;\r" expect "\nOk." expect "\n:)" -send -- "CREATE TABLE t01565 (c0 String, c1 Int32) ENGINE = Memory() ;\n" -expect "\r\n" -send -- "\r" +send -- "CREATE TABLE t01565 (c0 String, c1 Int32) ENGINE = Memory() ;\r" expect "\nOk." expect "\n:) " -send -- "INSERT INTO t01565(c0, c1) VALUES (\"1\",1) ;\n" -expect "\r\n" -send -- "\r" +send -- "INSERT INTO t01565(c0, c1) VALUES (\"1\",1) ;\r" expect "\n:) " -send -- "INSERT INTO t01565(c0, c1) VALUES ('1', 1) ;\n" -expect "\r\n" -send -- "\r" +send -- "INSERT INTO t01565(c0, c1) VALUES ('1', 1) ;\r" expect "\nOk." expect "\n:) " From 064acacd93a7de86cd66bf551905b9ff365a9eef Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Apr 2024 12:43:10 +0200 Subject: [PATCH 1075/1165] Update test --- .../0_stateless/02263_format_insert_settings.reference | 10 ---------- .../0_stateless/02263_format_insert_settings.sh | 2 -- 2 files changed, 12 deletions(-) diff --git a/tests/queries/0_stateless/02263_format_insert_settings.reference b/tests/queries/0_stateless/02263_format_insert_settings.reference index 2bba75f6788..ea8b78faf8c 100644 --- a/tests/queries/0_stateless/02263_format_insert_settings.reference +++ b/tests/queries/0_stateless/02263_format_insert_settings.reference @@ -21,10 +21,6 @@ INSERT INTO foo FORMAT Values INSERT INTO foo SELECT 1 [oneline] insert into foo select 1 INSERT INTO foo SELECT 1 -[multi] insert into foo watch bar -INSERT INTO foo WATCH bar -[oneline] insert into foo watch bar -INSERT INTO foo WATCH bar [multi] insert into foo format tsv INSERT INTO foo FORMAT tsv [oneline] insert into foo format tsv @@ -41,12 +37,6 @@ SETTINGS max_threads = 1 SELECT 1 [oneline] insert into foo settings max_threads=1 select 1 INSERT INTO foo SETTINGS max_threads = 1 SELECT 1 -[multi] insert into foo settings max_threads=1 watch bar -INSERT INTO foo -SETTINGS max_threads = 1 -WATCH bar -[oneline] insert into foo settings max_threads=1 watch bar -INSERT INTO foo SETTINGS max_threads = 1 WATCH bar [multi] insert into foo settings max_threads=1 format tsv INSERT INTO foo SETTINGS max_threads = 1 diff --git a/tests/queries/0_stateless/02263_format_insert_settings.sh b/tests/queries/0_stateless/02263_format_insert_settings.sh index 49aa56d6c0a..808ab23ee59 100755 --- a/tests/queries/0_stateless/02263_format_insert_settings.sh +++ b/tests/queries/0_stateless/02263_format_insert_settings.sh @@ -40,12 +40,10 @@ $CLICKHOUSE_CLIENT -q 'drop table data_02263' run_format_both 'insert into foo values' run_format_both 'insert into foo select 1' -run_format_both 'insert into foo watch bar' run_format_both 'insert into foo format tsv' run_format_both 'insert into foo settings max_threads=1 values' run_format_both 'insert into foo settings max_threads=1 select 1' -run_format_both 'insert into foo settings max_threads=1 watch bar' run_format_both 'insert into foo settings max_threads=1 format tsv' run_format_both 'insert into foo select 1 settings max_threads=1' run_format_both 'insert into foo settings max_threads=1 select 1 settings max_threads=1' From 9ebf091ae41c8fc9deda4a2fd6260dd8c47d102a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Apr 2024 13:03:55 +0200 Subject: [PATCH 1076/1165] Documentation --- src/Client/ClientBaseHelpers.cpp | 20 ++++++++++++++++++++ src/Parsers/IParser.h | 2 ++ 2 files changed, 22 insertions(+) diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index da164bd1226..3f3e3b1492f 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -97,9 +97,16 @@ void highlight(const String & query, std::vector & colors { using namespace replxx; + /// The `colors` array maps to a Unicode code point position in a string into a color. + /// A color is set for every position individually (not for a range). + + /// Empty input. if (colors.empty()) return; + /// The colors should be legible (and look gorgeous) in both dark and light themes. + /// When modifying this, check it in both themes. + static const std::unordered_map type_to_color = { {Highlight::keyword, replxx::color::bold(Replxx::Color::DEFAULT)}, @@ -111,11 +118,20 @@ void highlight(const String & query, std::vector & colors {Highlight::string, Replxx::Color::GREEN}, }; + /// We set reasonably small limits for size/depth, because we don't want the CLI to be slow. + /// While syntax highlighting is unneeded for long queries, which the user couldn't read anyway. + const char * begin = query.data(); const char * end = begin + query.size(); Tokens tokens(begin, end, 1000, true); IParser::Pos token_iterator(tokens, static_cast(1000), static_cast(10000)); Expected expected; + + /// We don't do highlighting for foreign dialects, such as PRQL and Kusto. + /// Only normal ClickHouse SQL queries are highlighted. + + /// Currently we highlight only the first query in the multi-query mode. + ParserQuery parser(end); ASTPtr ast; bool parse_res = false; @@ -138,6 +154,7 @@ void highlight(const String & query, std::vector & colors auto it = type_to_color.find(range.highlight); if (it != type_to_color.end()) { + /// We have to map from byte positions to Unicode positions. pos += UTF8::countCodePoints(reinterpret_cast(prev), range.begin - prev); size_t utf8_len = UTF8::countCodePoints(reinterpret_cast(range.begin), range.end - range.begin); @@ -153,6 +170,8 @@ void highlight(const String & query, std::vector & colors /// Raw data in INSERT queries, which is not necessarily tokenized. const char * insert_data = ast ? getInsertData(ast) : nullptr; + /// Highlight the last error in red. If the parser failed or the lexer found an invalid token, + /// or if it didn't parse all the data (except, the data for INSERT query, which is legitimately unparsed) if ((!parse_res || last_token.isError() || (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon)) && !(insert_data && expected.max_parsed_pos >= insert_data)) { @@ -164,6 +183,7 @@ void highlight(const String & query, std::vector & colors colors[pos] = Replxx::Color::BRIGHTRED; } + /// This is a callback for the client/local app to better find query end. Note: this is a kludge, remove it. if (last_token.type == TokenType::Semicolon || last_token.type == TokenType::VerticalDelimiter || query.ends_with(';') || query.ends_with("\\G")) /// This is for raw data in INSERT queries, which is not necessarily tokenized. { diff --git a/src/Parsers/IParser.h b/src/Parsers/IParser.h index d79bc0fb999..f8146c0a4f6 100644 --- a/src/Parsers/IParser.h +++ b/src/Parsers/IParser.h @@ -48,6 +48,8 @@ struct HighlightedRange /** Collects variants, how parser could proceed further at rightmost position. + * Also collects a mapping of parsed ranges for highlighting, + * which is accumulated through the parsing. */ struct Expected { From 444016fb3ee585cae98df9f16285b9c0fff6577a Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Mon, 8 Apr 2024 11:20:30 +0000 Subject: [PATCH 1077/1165] CI: fix unittest issue --- tests/ci/ci.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 36e9b183805..f60c40f5860 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -773,6 +773,7 @@ class CiOptions: not pr_info.is_pr() and not debug_message ): # if commit_message is provided it's test/debug scenario - do not return # CI options can be configured in PRs only + # if debug_message is provided - it's a test return res message = debug_message or GitRunner(set_cwd_to_git_root=True).run( f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1" @@ -790,9 +791,9 @@ class CiOptions: print(f"CI tags from PR body: [{matches_pr}]") matches = list(set(matches + matches_pr)) - if "do not test" in pr_info.labels: - # do_not_test could be set in GH labels - res.do_not_test = True + if "do not test" in pr_info.labels: + # do_not_test could be set in GH labels + res.do_not_test = True for match in matches: if match.startswith("job_"): From 094f94882c972a798ace493ca9a7019255a64f7b Mon Sep 17 00:00:00 2001 From: Ilya Andreev <18560147+andreev-io@users.noreply.github.com> Date: Mon, 8 Apr 2024 11:35:03 +0100 Subject: [PATCH 1078/1165] Fix a typo in the documentation of the ALTER TABLE ... MODIFY QUERY statement --- docs/en/sql-reference/statements/alter/view.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/view.md b/docs/en/sql-reference/statements/alter/view.md index 59045afdeb6..e063b27424e 100644 --- a/docs/en/sql-reference/statements/alter/view.md +++ b/docs/en/sql-reference/statements/alter/view.md @@ -8,7 +8,7 @@ sidebar_label: VIEW You can modify `SELECT` query that was specified when a [materialized view](../create/view.md#materialized) was created with the `ALTER TABLE … MODIFY QUERY` statement without interrupting ingestion process. -This command is created to change materialized view created with `TO [db.]name` clause. It does not change the structure of the underling storage table and it does not change the columns' definition of the materialized view, because of this the application of this command is very limited for materialized views are created without `TO [db.]name` clause. +This command is created to change materialized view created with `TO [db.]name` clause. It does not change the structure of the underlying storage table and it does not change the columns' definition of the materialized view, because of this the application of this command is very limited for materialized views are created without `TO [db.]name` clause. **Example with TO table** From 83d1f1a8769d3be8d78f48db82873b9438ac87f4 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Mon, 8 Apr 2024 11:51:59 +0000 Subject: [PATCH 1079/1165] CI: fix for docs only pr --- tests/ci/ci.py | 7 ++++--- tests/ci/pr_info.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 36e9b183805..f60c40f5860 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -773,6 +773,7 @@ class CiOptions: not pr_info.is_pr() and not debug_message ): # if commit_message is provided it's test/debug scenario - do not return # CI options can be configured in PRs only + # if debug_message is provided - it's a test return res message = debug_message or GitRunner(set_cwd_to_git_root=True).run( f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1" @@ -790,9 +791,9 @@ class CiOptions: print(f"CI tags from PR body: [{matches_pr}]") matches = list(set(matches + matches_pr)) - if "do not test" in pr_info.labels: - # do_not_test could be set in GH labels - res.do_not_test = True + if "do not test" in pr_info.labels: + # do_not_test could be set in GH labels + res.do_not_test = True for match in matches: if match.startswith("job_"): diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index ddf59c49e1f..204284785c9 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -26,6 +26,7 @@ NeedsDataType = Dict[str, Dict[str, Union[str, Dict[str, str]]]] DIFF_IN_DOCUMENTATION_EXT = [ ".html", ".md", + ".mdx", ".yml", ".txt", ".css", From 259d50c57b6227b2a078effcef8de19cd23c346f Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 8 Apr 2024 14:36:07 +0200 Subject: [PATCH 1080/1165] Add more comments --- src/Interpreters/Cache/EvictionCandidates.cpp | 22 ++++-- src/Interpreters/Cache/EvictionCandidates.h | 8 +- src/Interpreters/Cache/FileCache.cpp | 77 ++++++++++++------- src/Interpreters/Cache/FileCacheFactory.cpp | 8 +- .../Cache/LRUFileCachePriority.cpp | 2 +- src/Interpreters/Cache/Metadata.cpp | 72 +++++++++-------- 6 files changed, 112 insertions(+), 77 deletions(-) diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index f9f9bdfe662..d20ae77d720 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -17,6 +17,11 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +EvictionCandidates::EvictionCandidates() + : log(getLogger("EvictionCandidates")) +{ +} + EvictionCandidates::~EvictionCandidates() { /// Here `queue_entries_to_invalidate` contains queue entries @@ -64,8 +69,11 @@ void EvictionCandidates::add( void EvictionCandidates::removeQueueEntries(const CachePriorityGuard::Lock & lock) { - auto log = getLogger("EvictionCandidates"); + /// Remove queue entries of eviction candidates. + /// This will release space we consider to be hold for them. + LOG_TEST(log, "Will remove {} eviction candidates", size()); + for (const auto & [key, key_candidates] : candidates) { for (const auto & candidate : key_candidates.candidates) @@ -87,6 +95,7 @@ void EvictionCandidates::evict() auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::FilesystemCacheEvictMicroseconds); + /// If queue entries are already removed, then nothing to invalidate. if (!removed_queue_entries) queue_entries_to_invalidate.reserve(candidates_size); @@ -184,6 +193,12 @@ void EvictionCandidates::finalize( on_finalize.clear(); } +bool EvictionCandidates::needFinalize() const +{ + /// Do we need to call finalize()? + return !on_finalize.empty() || !queue_entries_to_invalidate.empty(); +} + void EvictionCandidates::setSpaceHolder( size_t size, size_t elements, @@ -196,9 +211,4 @@ void EvictionCandidates::setSpaceHolder( hold_space = std::make_unique(size, elements, priority, lock); } -void EvictionCandidates::insert(EvictionCandidates && other, const CachePriorityGuard::Lock &) -{ - candidates.insert(make_move_iterator(other.candidates.begin()), make_move_iterator(other.candidates.end())); -} - } diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index baacbc0cfae..0dcc6bc0dda 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -9,7 +9,7 @@ class EvictionCandidates : private boost::noncopyable public: using FinalizeEvictionFunc = std::function; - EvictionCandidates() = default; + EvictionCandidates(); ~EvictionCandidates(); void add( @@ -17,8 +17,6 @@ public: LockedKey & locked_key, const CachePriorityGuard::Lock &); - void insert(EvictionCandidates && other, const CachePriorityGuard::Lock &); - void evict(); void removeQueueEntries(const CachePriorityGuard::Lock &); @@ -29,6 +27,8 @@ public: FileCacheQueryLimit::QueryContext * query_context, const CachePriorityGuard::Lock &); + bool needFinalize() const; + size_t size() const { return candidates_size; } auto begin() const { return candidates.begin(); } @@ -57,6 +57,8 @@ private: bool removed_queue_entries = false; IFileCachePriority::HoldSpacePtr hold_space; + + LoggerPtr log; }; using EvictionCandidatesPtr = std::unique_ptr; diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 12ea2c178bc..29f2ebeca55 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1389,7 +1389,18 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, || new_settings.max_elements != actual_settings.max_elements) { EvictionCandidates eviction_candidates; - bool limits_satisfied = false; + bool modified_size_limit = false; + + /// In order to not block cache for the duration of cache resize, + /// we do: + /// a. Take a cache lock. + /// 1. Collect eviction candidates, + /// 2. Remove queue entries of eviction candidates. + /// This will release space we consider to be hold for them, + /// so that we can safely modify size limits. + /// 3. Modify size limits of cache. + /// b. Release a cache lock. + /// 1. Do actual eviction from filesystem. { cache_is_being_resized.store(true, std::memory_order_relaxed); SCOPE_EXIT({ @@ -1399,38 +1410,45 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, auto cache_lock = lockCache(); FileCacheReserveStat stat; - limits_satisfied = main_priority->collectCandidatesForEviction( - new_settings.max_size, new_settings.max_elements, 0/* max_candidates_to_evict */, stat, eviction_candidates, cache_lock); - - eviction_candidates.removeQueueEntries(cache_lock); - - if (limits_satisfied) + if (main_priority->collectCandidatesForEviction( + new_settings.max_size, new_settings.max_elements, 0/* max_candidates_to_evict */, + stat, eviction_candidates, cache_lock)) { + /// Remove only queue entries of eviction candidates. + eviction_candidates.removeQueueEntries(cache_lock); + /// Note that (in-memory) metadata about corresponding file segments + /// (e.g. file segment info in CacheMetadata) will be removed + /// only after eviction from filesystem. This is needed to avoid + /// a race on removal of file from filesystsem and + /// addition of the same file as part of a newly cached file segment. + + /// Modify cache size limits. + /// From this point cache eviction will follow them. main_priority->modifySizeLimits( - new_settings.max_size, new_settings.max_elements, new_settings.slru_size_ratio, cache_lock); + new_settings.max_size, new_settings.max_elements, + new_settings.slru_size_ratio, cache_lock); + + modified_size_limit = true; } - else + } + + if (modified_size_limit) + { + try { - LOG_WARNING(log, "Unable to modify size limit from {} to {}, " - "elements limit from {} to {}", - actual_settings.max_size, new_settings.max_size, - actual_settings.max_elements, new_settings.max_elements); + /// Do actual eviction from filesystem. + eviction_candidates.evict(); + } + catch (...) + { + if (eviction_candidates.needFinalize()) + eviction_candidates.finalize(nullptr, lockCache()); + throw; } - } - try - { - eviction_candidates.evict(); - } - catch (...) - { - auto cache_lock = lockCache(); - eviction_candidates.finalize(nullptr, cache_lock); - throw; - } + if (eviction_candidates.needFinalize()) + eviction_candidates.finalize(nullptr, lockCache()); - if (limits_satisfied) - { LOG_INFO(log, "Changed max_size from {} to {}, max_elements from {} to {}", actual_settings.max_size, new_settings.max_size, actual_settings.max_elements, new_settings.max_elements); @@ -1438,6 +1456,13 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, actual_settings.max_size = new_settings.max_size; actual_settings.max_elements = new_settings.max_elements; } + else + { + LOG_WARNING(log, "Unable to modify size limit from {} to {}, " + "elements limit from {} to {}", + actual_settings.max_size, new_settings.max_size, + actual_settings.max_elements, new_settings.max_elements); + } } if (new_settings.max_file_segment_size != actual_settings.max_file_segment_size) diff --git a/src/Interpreters/Cache/FileCacheFactory.cpp b/src/Interpreters/Cache/FileCacheFactory.cpp index 747b31bff64..a7a5834f03d 100644 --- a/src/Interpreters/Cache/FileCacheFactory.cpp +++ b/src/Interpreters/Cache/FileCacheFactory.cpp @@ -142,10 +142,8 @@ void FileCacheFactory::updateSettingsFromConfig(const Poco::Util::AbstractConfig caches_by_name_copy = caches_by_name; } - auto * log = &Poco::Logger::get("FileCacheFactory"); - std::unordered_set checked_paths; - for (const auto & [cache_name, cache_info] : caches_by_name_copy) + for (const auto & [_, cache_info] : caches_by_name_copy) { if (cache_info->config_path.empty() || checked_paths.contains(cache_info->config_path)) continue; @@ -158,10 +156,12 @@ void FileCacheFactory::updateSettingsFromConfig(const Poco::Util::AbstractConfig FileCacheSettings old_settings = cache_info->getSettings(); if (old_settings == new_settings) { - LOG_TRACE(log, "No settings changes for cache: {}", cache_name); continue; } + /// FIXME: registerDiskCache modifies `path` setting of FileCacheSettings if path is relative. + /// This can lead to calling applySettingsIfPossible even though nothing changed, which is avoidable. + // LOG_TRACE(log, "Will apply settings changes for cache {}. " // "Settings changes: {} (new settings: {}, old_settings: {})", // cache_name, fmt::join(new_settings.getSettingsDiff(old_settings), ", "), diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index e859529f5e7..1a2040f9ed2 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -280,7 +280,7 @@ bool LRUFileCachePriority::collectCandidatesForEviction( auto can_fit = [&] { - return canFit(size, 1, stat.total_stat.releasable_size, stat.total_stat.releasable_count, lock); + return canFit(size, elements, stat.total_stat.releasable_size, stat.total_stat.releasable_count, lock); }; iterateForEviction(res, stat, can_fit, lock); diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 631c1aa2ae6..2cbd56ba0bc 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -941,49 +941,47 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl( file_segment->detach(segment_lock, *this); + try { - try + const auto path = key_metadata->getFileSegmentPath(*file_segment); + if (file_segment->segment_kind == FileSegmentKind::Temporary) { - const auto path = key_metadata->getFileSegmentPath(*file_segment); - if (file_segment->segment_kind == FileSegmentKind::Temporary) - { - /// FIXME: For temporary file segment the requirement is not as strong because - /// the implementation of "temporary data in cache" creates files in advance. - if (fs::exists(path)) - fs::remove(path); - } - else if (file_segment->downloaded_size == 0) - { - chassert(!fs::exists(path)); - } - else if (fs::exists(path)) - { + /// FIXME: For temporary file segment the requirement is not as strong because + /// the implementation of "temporary data in cache" creates files in advance. + if (fs::exists(path)) fs::remove(path); - - /// Clear OpenedFileCache to avoid reading from incorrect file descriptor. - int flags = file_segment->getFlagsForLocalRead(); - /// Files are created with flags from file_segment->getFlagsForLocalRead() - /// plus optionally O_DIRECT is added, depends on query setting, so remove both. - OpenedFileCache::instance().remove(path, flags); - OpenedFileCache::instance().remove(path, flags | O_DIRECT); - - LOG_TEST(key_metadata->logger(), "Removed file segment at path: {}", path); - } - else if (!can_be_broken) - { -#ifdef ABORT_ON_LOGICAL_ERROR - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected path {} to exist", path); -#else - LOG_WARNING(key_metadata->logger(), "Expected path {} to exist, while removing {}:{}", - path, getKey(), file_segment->offset()); -#endif - } } - catch (...) + else if (file_segment->downloaded_size == 0) { - tryLogCurrentException(__PRETTY_FUNCTION__); - chassert(false); + chassert(!fs::exists(path)); } + else if (fs::exists(path)) + { + fs::remove(path); + + /// Clear OpenedFileCache to avoid reading from incorrect file descriptor. + int flags = file_segment->getFlagsForLocalRead(); + /// Files are created with flags from file_segment->getFlagsForLocalRead() + /// plus optionally O_DIRECT is added, depends on query setting, so remove both. + OpenedFileCache::instance().remove(path, flags); + OpenedFileCache::instance().remove(path, flags | O_DIRECT); + + LOG_TEST(key_metadata->logger(), "Removed file segment at path: {}", path); + } + else if (!can_be_broken) + { +#ifdef ABORT_ON_LOGICAL_ERROR + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected path {} to exist", path); +#else + LOG_WARNING(key_metadata->logger(), "Expected path {} to exist, while removing {}:{}", + path, getKey(), file_segment->offset()); +#endif + } + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + chassert(false); } return key_metadata->erase(it); From b6eef6137823a92c6a6ba601bb4c879b2dee30fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 5 Apr 2024 17:10:01 +0200 Subject: [PATCH 1081/1165] Fix one phony case --- contrib/avro-cmake/CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt index 63b3854eef9..c99c7dd4624 100644 --- a/contrib/avro-cmake/CMakeLists.txt +++ b/contrib/avro-cmake/CMakeLists.txt @@ -62,9 +62,12 @@ target_link_libraries (_avrocpp PRIVATE ch_contrib::snappy) # create a symlink to include headers with set(AVRO_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include") -ADD_CUSTOM_TARGET(avro_symlink_headers ALL - COMMAND ${CMAKE_COMMAND} -E make_directory "${AVRO_INCLUDE_DIR}" - COMMAND ${CMAKE_COMMAND} -E create_symlink "${AVROCPP_ROOT_DIR}/api" "${AVRO_INCLUDE_DIR}/avro" +ADD_CUSTOM_COMMAND(OUTPUT "${AVRO_INCLUDE_DIR}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${AVRO_INCLUDE_DIR}" + COMMAND ${CMAKE_COMMAND} -E create_symlink "${AVROCPP_ROOT_DIR}/api" "${AVRO_INCLUDE_DIR}/avro" + DEPENDS "${AVROCPP_ROOT_DIR}/api" ) +ADD_CUSTOM_TARGET(avro_symlink_headers ALL + DEPENDS "${AVRO_INCLUDE_DIR}") add_dependencies(_avrocpp avro_symlink_headers) target_include_directories(_avrocpp SYSTEM BEFORE PUBLIC "${AVRO_INCLUDE_DIR}") From 31cd71f8f2e78623af980d14ab0b29b3969757da Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 8 Apr 2024 14:39:11 +0200 Subject: [PATCH 1082/1165] Rename test --- ...ence => 03032_dynamically_resize_filesystem_cache_2.reference} | 0 ...hardcore.sh => 03032_dynamically_resize_filesystem_cache_2.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{03032_dynamically_resize_filesystem_cache_hardcore.reference => 03032_dynamically_resize_filesystem_cache_2.reference} (100%) rename tests/queries/0_stateless/{03032_dynamically_resize_filesystem_cache_hardcore.sh => 03032_dynamically_resize_filesystem_cache_2.sh} (100%) diff --git a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.reference b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.reference similarity index 100% rename from tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.reference rename to tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.reference diff --git a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.sh b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh similarity index 100% rename from tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_hardcore.sh rename to tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh From e4b0ca5d836e14fada7592777a0443914bfbaa47 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 8 Apr 2024 12:59:20 +0000 Subject: [PATCH 1083/1165] Fix filter pushdown from additional_table_filters in Merge engine in analyzer --- src/Planner/PlannerJoinTree.cpp | 3 ++- src/Storages/StorageDummy.h | 6 ++++++ ...03033_analyzer_merge_engine_filter_push_down.reference | 3 +++ .../03033_analyzer_merge_engine_filter_push_down.sql | 8 ++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.reference create mode 100644 tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.sql diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index d2f37ff1ad4..534080f1739 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -814,7 +814,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres bool optimize_move_to_prewhere = settings.optimize_move_to_prewhere && (!is_final || settings.optimize_move_to_prewhere_if_final); - if (storage->supportsPrewhere() && optimize_move_to_prewhere) + auto supported_prewhere_columns = storage->supportedPrewhereColumns(); + if (storage->canMoveConditionsToPrewhere() && optimize_move_to_prewhere && (!supported_prewhere_columns || supported_prewhere_columns->contains(filter_info.column_name))) { if (!prewhere_info) prewhere_info = std::make_shared(); diff --git a/src/Storages/StorageDummy.h b/src/Storages/StorageDummy.h index e9d8f90f755..ae9bf2483e1 100644 --- a/src/Storages/StorageDummy.h +++ b/src/Storages/StorageDummy.h @@ -19,6 +19,12 @@ public: bool supportsSampling() const override { return true; } bool supportsFinal() const override { return true; } bool supportsPrewhere() const override { return true; } + + std::optional supportedPrewhereColumns() const override + { + return original_storage_snapshot ? original_storage_snapshot->storage.supportedPrewhereColumns() : std::nullopt; + } + bool supportsSubcolumns() const override { return true; } bool supportsDynamicSubcolumns() const override { return true; } bool canMoveConditionsToPrewhere() const override diff --git a/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.reference b/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.reference new file mode 100644 index 00000000000..86a00059854 --- /dev/null +++ b/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.reference @@ -0,0 +1,3 @@ +UInt32 1 +UInt32 2 +UInt32 3 diff --git a/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.sql b/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.sql new file mode 100644 index 00000000000..9be1152bbbf --- /dev/null +++ b/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.sql @@ -0,0 +1,8 @@ +set allow_suspicious_low_cardinality_types=1; +drop table if exists test; +create table test (`x` LowCardinality(Nullable(UInt32)), `y` String) engine = MergeTree order by tuple(); +insert into test values (1, 'a'), (2, 'bb'), (3, 'ccc'), (4, 'dddd'); +create table m_table (x UInt32, y String) engine = Merge(currentDatabase(), 'test*'); +select toTypeName(x), x FROM m_table SETTINGS additional_table_filters = {'m_table':'x != 4'}, optimize_move_to_prewhere=1, allow_experimental_analyzer=1; +drop table test; + From 5e87ecf32e9138c1cf9e249f65580dd9a9c5732c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 8 Apr 2024 12:59:59 +0000 Subject: [PATCH 1084/1165] Revert "Updating test." This reverts commit 1a75d3ed46d0a2fe6a596afbc27ae20ff97427fb. --- .../02802_with_cube_with_totals.reference | 30 ------------------- .../02802_with_cube_with_totals.sql | 1 - 2 files changed, 31 deletions(-) diff --git a/tests/queries/0_stateless/02802_with_cube_with_totals.reference b/tests/queries/0_stateless/02802_with_cube_with_totals.reference index 206c32e562b..c7b7b570456 100644 --- a/tests/queries/0_stateless/02802_with_cube_with_totals.reference +++ b/tests/queries/0_stateless/02802_with_cube_with_totals.reference @@ -1,35 +1,5 @@ ((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 ((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 -((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 ((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 \N diff --git a/tests/queries/0_stateless/02802_with_cube_with_totals.sql b/tests/queries/0_stateless/02802_with_cube_with_totals.sql index 168e4d61b68..77adb68eb4b 100644 --- a/tests/queries/0_stateless/02802_with_cube_with_totals.sql +++ b/tests/queries/0_stateless/02802_with_cube_with_totals.sql @@ -1,3 +1,2 @@ -set allow_experimental_analyzer=1; SELECT tuple((2147483648, (-0., 1.1754943508222875e-38, 2147483646, '-9223372036854775808', NULL))), toInt128(0.0001) GROUP BY ((256, toInt64(1.1754943508222875e-38), NULL), NULL, -0., ((65535, '-92233720368547758.07'), 0.9999), tuple(((1., 3.4028234663852886e38, '1', 0.5), NULL, tuple('0.1')))) WITH CUBE WITH TOTALS; SELECT NULL GROUP BY toUUID(NULL, '0', NULL, '0.0000065535'), 1 WITH CUBE WITH TOTALS; From 5e1c1b6b94d920fbfb361c0cf606728f730e149a Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Mon, 8 Apr 2024 13:41:44 +0000 Subject: [PATCH 1085/1165] CI: test merge queue --- tests/ci/ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index f60c40f5860..c2962c5b40e 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -318,7 +318,7 @@ class CiCache: self.update() if self.cache_data_fetched: - # there are no record w/o underling data - no need to fetch + # there are no records without fetched data - no need to fetch return self # clean up From 44d3612d77032e2b104296840690154e53d8f073 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 8 Apr 2024 15:52:38 +0200 Subject: [PATCH 1086/1165] Review suggestion --- src/Interpreters/Cache/FileCache.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 29f2ebeca55..be452e43bed 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -1458,10 +1458,13 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, } else { - LOG_WARNING(log, "Unable to modify size limit from {} to {}, " - "elements limit from {} to {}", - actual_settings.max_size, new_settings.max_size, - actual_settings.max_elements, new_settings.max_elements); + LOG_WARNING( + log, "Unable to modify size limit from {} to {}, elements limit from {} to {}. " + "`max_size` and `max_elements` settings will remain inconsistent with config.xml. " + "Next attempt to update them will happen on the next config reload. " + "You can trigger it with SYSTEM RELOAD CONFIG.", + actual_settings.max_size, new_settings.max_size, + actual_settings.max_elements, new_settings.max_elements); } } From 8ac9cbd80d581183edd7add2417008f531e28656 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 8 Apr 2024 14:13:43 +0000 Subject: [PATCH 1087/1165] Add part name to check part exception message --- src/Storages/MergeTree/DataPartsExchange.cpp | 4 +-- .../MergeTree/MergeTreeDataPartChecksum.cpp | 33 ++++++++++--------- .../MergeTree/MergeTreeDataPartChecksum.h | 8 ++--- .../ReplicatedMergeTreePartCheckThread.cpp | 2 +- src/Storages/MergeTree/checkDataPart.cpp | 2 +- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 7 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 91444d76a52..c56530b97eb 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -318,7 +318,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( } if (!from_remote_disk && isFullPartStorage(part->getDataPartStorage())) - part->checksums.checkEqual(data_checksums, false); + part->checksums.checkEqual(data_checksums, false, part->name); return data_checksums; } @@ -906,7 +906,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( else { if (isFullPartStorage(new_data_part->getDataPartStorage())) - new_data_part->checksums.checkEqual(data_checksums, false); + new_data_part->checksums.checkEqual(data_checksums, false, new_data_part->name); LOG_DEBUG(log, "Download of part {} onto disk {} finished.", part_name, disk->getName()); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index d60f4cc7354..7c9e4a371ab 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -28,33 +28,34 @@ namespace ErrorCodes } -void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const +void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name, const String & part_name) const { if (is_compressed && have_uncompressed) { if (!rhs.is_compressed) - throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, "No uncompressed checksum for file {}", name); + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, "No uncompressed checksum for file {}, data part {}", name, part_name); + if (rhs.uncompressed_size != uncompressed_size) { - throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected uncompressed size of file {} in data part ({} vs {})", - name, uncompressed_size, rhs.uncompressed_size); + throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected uncompressed size of file {} in data part {} ({} vs {})", + name, part_name, uncompressed_size, rhs.uncompressed_size); } if (rhs.uncompressed_hash != uncompressed_hash) { - throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, "Checksum mismatch for uncompressed file {} in data part ({} vs {})", - name, getHexUIntLowercase(uncompressed_hash), getHexUIntLowercase(rhs.uncompressed_hash)); + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, "Checksum mismatch for uncompressed file {} in data part {} ({} vs {})", + name, part_name, getHexUIntLowercase(uncompressed_hash), getHexUIntLowercase(rhs.uncompressed_hash)); } return; } if (rhs.file_size != file_size) { - throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {} in data part ({} vs {})", - name, file_size, rhs.file_size); + throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {} in data part {} ({} vs {})", + name, part_name, file_size, rhs.file_size); } if (rhs.file_hash != file_hash) { - throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, "Checksum mismatch for file {} in data part ({} vs {})", - name, getHexUIntLowercase(file_hash), getHexUIntLowercase(rhs.file_hash)); + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, "Checksum mismatch for file {} in data part {} ({} vs {})", + name, part_name, getHexUIntLowercase(file_hash), getHexUIntLowercase(rhs.file_hash)); } } @@ -79,7 +80,7 @@ void MergeTreeDataPartChecksum::checkSize(const IDataPartStorage & storage, cons } -void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const +void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed, const String & part_name) const { for (const auto & [name, _] : rhs.files) if (!files.contains(name)) @@ -95,7 +96,7 @@ void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & r if (it == rhs.files.end()) throw Exception(ErrorCodes::NO_FILE_IN_DATA_PART, "No file {} in data part", name); - checksum.checkEqual(it->second, have_uncompressed, name); + checksum.checkEqual(it->second, have_uncompressed, name, part_name); } } @@ -435,19 +436,19 @@ String MinimalisticDataPartChecksums::getSerializedString(const MergeTreeDataPar return checksums.getSerializedString(); } -void MinimalisticDataPartChecksums::checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) const +void MinimalisticDataPartChecksums::checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files, const String & part_name) const { if (full_checksums && rhs.full_checksums) - full_checksums->checkEqual(*rhs.full_checksums, check_uncompressed_hash_in_compressed_files); + full_checksums->checkEqual(*rhs.full_checksums, check_uncompressed_hash_in_compressed_files, part_name); // If full checksums were checked, check total checksums just in case checkEqualImpl(rhs, check_uncompressed_hash_in_compressed_files); } -void MinimalisticDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) const +void MinimalisticDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files, const String & part_name) const { if (full_checksums) - full_checksums->checkEqual(rhs, check_uncompressed_hash_in_compressed_files); + full_checksums->checkEqual(rhs, check_uncompressed_hash_in_compressed_files, part_name); // If full checksums were checked, check total checksums just in case MinimalisticDataPartChecksums rhs_minimalistic; diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h index d4980a67a43..05178dc3a60 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -32,7 +32,7 @@ struct MergeTreeDataPartChecksum : file_size(file_size_), file_hash(file_hash_), is_compressed(true), uncompressed_size(uncompressed_size_), uncompressed_hash(uncompressed_hash_) {} - void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const; + void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name, const String & part_name) const; void checkSize(const IDataPartStorage & storage, const String & name) const; }; @@ -61,7 +61,7 @@ struct MergeTreeDataPartChecksums /// Checks that the set of columns and their checksums are the same. If not, throws an exception. /// If have_uncompressed, for compressed files it compares the checksums of the decompressed data. /// Otherwise, it compares only the checksums of the files. - void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const; + void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed, const String & part_name) const; static bool isBadChecksumsErrorCode(int code); @@ -132,8 +132,8 @@ struct MinimalisticDataPartChecksums String getSerializedString() const; static String getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic); - void checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) const; - void checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) const; + void checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files, const String & part_name) const; + void checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files, const String & part_name) const; void checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) const; }; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 181f54688f9..d7601e6e638 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -359,7 +359,7 @@ ReplicatedCheckResult ReplicatedMergeTreePartCheckThread::checkPartImpl(const St if (local_part_header.getColumnsHash() != zk_part_header.getColumnsHash()) throw Exception(ErrorCodes::TABLE_DIFFERS_TOO_MUCH, "Columns of local part {} are different from ZooKeeper", part_name); - zk_part_header.getChecksums().checkEqual(local_part_header.getChecksums(), true); + zk_part_header.getChecksums().checkEqual(local_part_header.getChecksums(), true, part_name); checkDataPart( part, diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index d64568e0c3e..81de9d5a9a1 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -350,7 +350,7 @@ static IMergeTreeDataPart::Checksums checkDataPart( return {}; if (require_checksums || !checksums_txt.files.empty()) - checksums_txt.checkEqual(checksums_data, check_uncompressed); + checksums_txt.checkEqual(checksums_data, check_uncompressed, data_part->name); return checksums_data; } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index c9f451b6bb1..f7b6d6d3c2a 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2319,7 +2319,7 @@ std::optional StorageMergeTree::checkDataNext(DataValidationTasksPt try { auto calculated_checksums = checkDataPart(part, false, noop, /* is_cancelled */[]{ return false; }, /* throw_on_broken_projection */true); - calculated_checksums.checkEqual(part->checksums, true); + calculated_checksums.checkEqual(part->checksums, true, part->name); auto & part_mutable = const_cast(*part); part_mutable.writeChecksums(part->checksums, local_context->getWriteSettings()); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ddb90f06679..f3547ec5b56 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1837,7 +1837,7 @@ bool StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps( "(it may rarely happen on race condition with KILL MUTATION).", part_name, replica); } - replica_part_header.getChecksums().checkEqual(local_part_header.getChecksums(), true); + replica_part_header.getChecksums().checkEqual(local_part_header.getChecksums(), true, part_name); break; } From f3dc77ee00008f640b4d1f47445a223bbe286000 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Mon, 8 Apr 2024 14:15:49 +0000 Subject: [PATCH 1088/1165] disable autofix for merge queue --- tests/ci/style_check.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 373fa7b316f..4580f007606 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -131,6 +131,11 @@ def main(): temp_path.mkdir(parents=True, exist_ok=True) pr_info = PRInfo() + + if pr_info.is_merge_queue() and args.push: + print("Auto style fix will be disabled for Merge Queue workflow") + args.push = False + run_cpp_check = True run_shell_check = True run_python_check = True From 82b2adef97a87de683c1a20ec696c03216416a23 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 8 Apr 2024 14:50:13 +0000 Subject: [PATCH 1089/1165] Fix GLOBAL IN table queries with analyzer. --- src/Planner/CollectSets.cpp | 52 +++++++++++-------- src/Planner/CollectSets.h | 4 ++ src/Storages/buildQueryTreeForShard.cpp | 7 ++- .../test_cluster_all_replicas/test.py | 11 ++++ 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index e150b8a5956..37502828f63 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -24,6 +24,34 @@ namespace ErrorCodes extern const int UNSUPPORTED_METHOD; } +QueryTreeNodePtr makeExecutableSubqueryForIn(const QueryTreeNodePtr & in_second_argument, const ContextPtr & context) +{ + auto subquery_to_execute = in_second_argument; + if (auto * table_node = in_second_argument->as()) + { + auto storage_snapshot = table_node->getStorageSnapshot(); + auto columns_to_select = storage_snapshot->getColumns(GetColumnsOptions(GetColumnsOptions::Ordinary)); + size_t columns_to_select_size = columns_to_select.size(); + auto column_nodes_to_select = std::make_shared(); + column_nodes_to_select->getNodes().reserve(columns_to_select_size); + NamesAndTypes projection_columns; + projection_columns.reserve(columns_to_select_size); + for (auto & column : columns_to_select) + { + column_nodes_to_select->getNodes().emplace_back(std::make_shared(column, subquery_to_execute)); + projection_columns.emplace_back(column.name, column.type); + } + auto subquery_for_table = std::make_shared(Context::createCopy(context)); + subquery_for_table->setIsSubquery(true); + subquery_for_table->getProjectionNode() = std::move(column_nodes_to_select); + subquery_for_table->getJoinTree() = std::move(subquery_to_execute); + subquery_for_table->resolveProjectionColumns(std::move(projection_columns)); + subquery_to_execute = std::move(subquery_for_table); + } + + return subquery_to_execute; +} + namespace { @@ -88,29 +116,7 @@ public: if (sets.findSubquery(set_key)) return; - auto subquery_to_execute = in_second_argument; - if (auto * table_node = in_second_argument->as()) - { - auto storage_snapshot = table_node->getStorageSnapshot(); - auto columns_to_select = storage_snapshot->getColumns(GetColumnsOptions(GetColumnsOptions::Ordinary)); - size_t columns_to_select_size = columns_to_select.size(); - auto column_nodes_to_select = std::make_shared(); - column_nodes_to_select->getNodes().reserve(columns_to_select_size); - NamesAndTypes projection_columns; - projection_columns.reserve(columns_to_select_size); - for (auto & column : columns_to_select) - { - column_nodes_to_select->getNodes().emplace_back(std::make_shared(column, subquery_to_execute)); - projection_columns.emplace_back(column.name, column.type); - } - auto subquery_for_table = std::make_shared(Context::createCopy(planner_context.getQueryContext())); - subquery_for_table->setIsSubquery(true); - subquery_for_table->getProjectionNode() = std::move(column_nodes_to_select); - subquery_for_table->getJoinTree() = std::move(subquery_to_execute); - subquery_for_table->resolveProjectionColumns(std::move(projection_columns)); - subquery_to_execute = std::move(subquery_for_table); - } - + auto subquery_to_execute = makeExecutableSubqueryForIn(in_second_argument, planner_context.getQueryContext()); sets.addFromSubquery(set_key, std::move(subquery_to_execute), settings); } else diff --git a/src/Planner/CollectSets.h b/src/Planner/CollectSets.h index 5f9f7a5a466..e4168c7dd49 100644 --- a/src/Planner/CollectSets.h +++ b/src/Planner/CollectSets.h @@ -14,4 +14,8 @@ struct SelectQueryOptions; */ void collectSets(const QueryTreeNodePtr & node, PlannerContext & planner_context); +/// Build subqiery which we execute for IN function. +/// It is needed to support `IN table` case. +QueryTreeNodePtr makeExecutableSubqueryForIn(const QueryTreeNodePtr & in_second_argument, const ContextPtr & context); + } diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 5284f52a7e4..09e48a93df4 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -361,10 +362,12 @@ QueryTreeNodePtr buildQueryTreeForShard(const PlannerContextPtr & planner_contex { auto & in_function_subquery_node = in_function_node->getArguments().getNodes().at(1); auto in_function_node_type = in_function_subquery_node->getNodeType(); - if (in_function_node_type != QueryTreeNodeType::QUERY && in_function_node_type != QueryTreeNodeType::UNION) + if (in_function_node_type != QueryTreeNodeType::QUERY && in_function_node_type != QueryTreeNodeType::UNION && in_function_node_type != QueryTreeNodeType::TABLE) continue; - auto temporary_table_expression_node = executeSubqueryNode(in_function_subquery_node, + auto subquery_to_execute = makeExecutableSubqueryForIn(in_function_subquery_node, planner_context->getQueryContext()); + + auto temporary_table_expression_node = executeSubqueryNode(subquery_to_execute, planner_context->getMutableQueryContext(), global_in_or_join_node.subquery_depth); diff --git a/tests/integration/test_cluster_all_replicas/test.py b/tests/integration/test_cluster_all_replicas/test.py index eb406de6a8d..15f3f36f74e 100644 --- a/tests/integration/test_cluster_all_replicas/test.py +++ b/tests/integration/test_cluster_all_replicas/test.py @@ -42,6 +42,17 @@ def test_cluster(start_cluster): ) +def test_global_in(start_cluster): + + node1.query("CREATE TABLE u(uid Int16) ENGINE=Log as select 0"); + + assert set( + node1.query( + """SELECT hostName(), * FROM clusterAllReplicas("one_shard_two_nodes", system.one) where dummy GLOBAL IN u""" + ).splitlines() + ) == {"node1\t0", "node2\t0"} + + @pytest.mark.parametrize( "cluster", [ From bcccbe2b19bd50429ce6de1f4e0619c0677fa4a1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 8 Apr 2024 14:54:36 +0000 Subject: [PATCH 1090/1165] Fixing typo --- src/Planner/CollectSets.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Planner/CollectSets.h b/src/Planner/CollectSets.h index e4168c7dd49..0ee006f3320 100644 --- a/src/Planner/CollectSets.h +++ b/src/Planner/CollectSets.h @@ -14,7 +14,7 @@ struct SelectQueryOptions; */ void collectSets(const QueryTreeNodePtr & node, PlannerContext & planner_context); -/// Build subqiery which we execute for IN function. +/// Build subquery which we execute for IN function. /// It is needed to support `IN table` case. QueryTreeNodePtr makeExecutableSubqueryForIn(const QueryTreeNodePtr & in_second_argument, const ContextPtr & context); From 7fcfbffd0cf215e87d36aa515d832577256ce2e3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 8 Apr 2024 15:01:11 +0000 Subject: [PATCH 1091/1165] Automatic style fix --- tests/integration/test_cluster_all_replicas/test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/test_cluster_all_replicas/test.py b/tests/integration/test_cluster_all_replicas/test.py index 15f3f36f74e..59b41ca87af 100644 --- a/tests/integration/test_cluster_all_replicas/test.py +++ b/tests/integration/test_cluster_all_replicas/test.py @@ -43,8 +43,7 @@ def test_cluster(start_cluster): def test_global_in(start_cluster): - - node1.query("CREATE TABLE u(uid Int16) ENGINE=Log as select 0"); + node1.query("CREATE TABLE u(uid Int16) ENGINE=Log as select 0") assert set( node1.query( From 603824748d7bec40d8dc7b30a33a988e214c3328 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Mon, 8 Apr 2024 15:03:13 +0000 Subject: [PATCH 1092/1165] CI: disable finish check for mq --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ff0adee1443..74ce8452de8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -157,7 +157,7 @@ jobs: ################################# Stage Final ################################# # FinishCheck: - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && github.event_name != 'merge_group' }} needs: [Tests_1, Tests_2] runs-on: [self-hosted, style-checker] steps: From f9402f3584aab08aeacaed8c278a8c26a526624e Mon Sep 17 00:00:00 2001 From: flynn Date: Mon, 8 Apr 2024 15:15:45 +0000 Subject: [PATCH 1093/1165] Fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 111 +++++++++++----------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index e29e8f2f136..48f32652198 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2275,57 +2275,62 @@ void QueryAnalyzer::mergeWindowWithParentWindow(const QueryTreeNodePtr & window_ */ void QueryAnalyzer::replaceNodesWithPositionalArguments(QueryTreeNodePtr & node_list, const QueryTreeNodes & projection_nodes, IdentifierResolveScope & scope) { - auto & node_list_typed = node_list->as(); - - for (auto & node : node_list_typed.getNodes()) + const auto & settings = scope.context->getSettingsRef(); + if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) { - auto * node_to_replace = &node; + auto & node_list_typed = node_list->as(); - if (auto * sort_node = node->as()) - node_to_replace = &sort_node->getExpression(); - - auto * constant_node = (*node_to_replace)->as(); - - if (!constant_node - || (constant_node->getValue().getType() != Field::Types::UInt64 && constant_node->getValue().getType() != Field::Types::Int64)) - continue; - - UInt64 pos; - if (constant_node->getValue().getType() == Field::Types::UInt64) + for (auto & node : node_list_typed.getNodes()) { - pos = constant_node->getValue().get(); - } - else // Int64 - { - auto value = constant_node->getValue().get(); - if (value > 0) - pos = value; - else + auto * node_to_replace = &node; + + if (auto * sort_node = node->as()) + node_to_replace = &sort_node->getExpression(); + + auto * constant_node = (*node_to_replace)->as(); + + if (!constant_node + || (constant_node->getValue().getType() != Field::Types::UInt64 + && constant_node->getValue().getType() != Field::Types::Int64)) + continue; + + UInt64 pos; + if (constant_node->getValue().getType() == Field::Types::UInt64) { - if (static_cast(std::abs(value)) > projection_nodes.size()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Negative positional argument number {} is out of bounds. Expected in range [-{}, -1]. In scope {}", - value, - projection_nodes.size(), - scope.scope_node->formatASTForErrorMessage()); - pos = projection_nodes.size() + value + 1; + pos = constant_node->getValue().get(); + } + else // Int64 + { + auto value = constant_node->getValue().get(); + if (value > 0) + pos = value; + else + { + if (static_cast(std::abs(value)) > projection_nodes.size()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Negative positional argument number {} is out of bounds. Expected in range [-{}, -1]. In scope {}", + value, + projection_nodes.size(), + scope.scope_node->formatASTForErrorMessage()); + pos = projection_nodes.size() + value + 1; + } } - } - if (!pos || pos > projection_nodes.size()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Positional argument number {} is out of bounds. Expected in range [1, {}]. In scope {}", - pos, - projection_nodes.size(), - scope.scope_node->formatASTForErrorMessage()); + if (!pos || pos > projection_nodes.size()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Positional argument number {} is out of bounds. Expected in range [1, {}]. In scope {}", + pos, + projection_nodes.size(), + scope.scope_node->formatASTForErrorMessage()); - --pos; - *node_to_replace = projection_nodes[pos]->clone(); - if (auto it = resolved_expressions.find(projection_nodes[pos]); it != resolved_expressions.end()) - { - resolved_expressions[*node_to_replace] = it->second; + --pos; + *node_to_replace = projection_nodes[pos]->clone(); + if (auto it = resolved_expressions.find(projection_nodes[pos]); it != resolved_expressions.end()) + { + resolved_expressions[*node_to_replace] = it->second; + } } } } @@ -6674,15 +6679,12 @@ void expandTuplesInList(QueryTreeNodes & key_list) */ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope) { - const auto & settings = scope.context->getSettingsRef(); - if (query_node_typed.isGroupByWithGroupingSets()) { QueryTreeNodes nullable_group_by_keys; for (auto & grouping_sets_keys_list_node : query_node_typed.getGroupBy().getNodes()) { - if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) - replaceNodesWithPositionalArguments(grouping_sets_keys_list_node, query_node_typed.getProjection().getNodes(), scope); + replaceNodesWithPositionalArguments(grouping_sets_keys_list_node, query_node_typed.getProjection().getNodes(), scope); // Remove redundant calls to `tuple` function. It simplifies checking if expression is an aggregation key. // It's required to support queries like: SELECT number FROM numbers(3) GROUP BY (number, number % 2) @@ -6701,8 +6703,7 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR } else { - if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) - replaceNodesWithPositionalArguments(query_node_typed.getGroupByNode(), query_node_typed.getProjection().getNodes(), scope); + replaceNodesWithPositionalArguments(query_node_typed.getGroupByNode(), query_node_typed.getProjection().getNodes(), scope); // Remove redundant calls to `tuple` function. It simplifies checking if expression is an aggregation key. // It's required to support queries like: SELECT number FROM numbers(3) GROUP BY (number, number % 2) @@ -7853,8 +7854,6 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.isCTE()) cte_in_resolve_process.insert(query_node_typed.getCTEName()); - const auto & settings = scope.context->getSettingsRef(); - bool is_rollup_or_cube = query_node_typed.isGroupByWithRollup() || query_node_typed.isGroupByWithCube(); if (query_node_typed.isGroupByWithGroupingSets() @@ -8038,8 +8037,9 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.hasOrderBy()) { - if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) - replaceNodesWithPositionalArguments(query_node_typed.getOrderByNode(), query_node_typed.getProjection().getNodes(), scope); + replaceNodesWithPositionalArguments(query_node_typed.getOrderByNode(), query_node_typed.getProjection().getNodes(), scope); + + const auto & settings = scope.context->getSettingsRef(); expandOrderByAll(query_node_typed, settings); resolveSortNodeList(query_node_typed.getOrderByNode(), scope); @@ -8062,8 +8062,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.hasLimitBy()) { - if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) - replaceNodesWithPositionalArguments(query_node_typed.getLimitByNode(), query_node_typed.getProjection().getNodes(), scope); + replaceNodesWithPositionalArguments(query_node_typed.getLimitByNode(), query_node_typed.getProjection().getNodes(), scope); resolveExpressionNodeList(query_node_typed.getLimitByNode(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); } From c69d8e18f245b67057f9c77c1c5e06352c805529 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 8 Apr 2024 15:38:15 +0000 Subject: [PATCH 1094/1165] Dont use constant database name --- ...unknown_identifier_materialized_column.sql | 10 ++++---- .../0_stateless/03053_analyzer_join_alias.sql | 16 ++++++------- ...same_table_name_in_different_databases.sql | 24 +++++++++---------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql index 276e4845831..938f270b9e4 100644 --- a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql +++ b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql @@ -1,14 +1,14 @@ -- https://github.com/ClickHouse/ClickHouse/issues/54317 SET allow_experimental_analyzer=1; -DROP DATABASE IF EXISTS 03049_database; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier}; -CREATE DATABASE 03049_database; -USE 03049_database; +CREATE DATABASE {CLICKHOUSE_DATABASE:Identifier}; +USE {CLICKHOUSE_DATABASE:Identifier}; CREATE TABLE l (y String) Engine Memory; CREATE TABLE r (d Date, y String, ty UInt16 MATERIALIZED toYear(d)) Engine Memory; select * from l L left join r R on L.y = R.y where R.ty >= 2019; select * from l left join r on l.y = r.y where r.ty >= 2019; -select * from 03049_database.l left join 03049_database.r on l.y = r.y where r.ty >= 2019; +select * from {CLICKHOUSE_DATABASE:Identifier}.l left join {CLICKHOUSE_DATABASE:Identifier}.r on l.y = r.y where r.ty >= 2019; -DROP DATABASE IF EXISTS 03049_database; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier}; diff --git a/tests/queries/0_stateless/03053_analyzer_join_alias.sql b/tests/queries/0_stateless/03053_analyzer_join_alias.sql index ef51ec73026..894b8af7c6f 100644 --- a/tests/queries/0_stateless/03053_analyzer_join_alias.sql +++ b/tests/queries/0_stateless/03053_analyzer_join_alias.sql @@ -1,9 +1,9 @@ -- https://github.com/ClickHouse/ClickHouse/issues/23104 SET allow_experimental_analyzer=1; -DROP DATABASE IF EXISTS test_03053; -CREATE DATABASE test_03053; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE:Identifier}; -CREATE TABLE test_03053.base +CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.base ( `id` UInt64, `id2` UInt64, @@ -14,7 +14,7 @@ ENGINE=MergeTree() PARTITION BY d ORDER BY (id,id2,d); -CREATE TABLE test_03053.derived1 +CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.derived1 ( `id1` UInt64, `d1` UInt64, @@ -24,7 +24,7 @@ ENGINE = MergeTree() PARTITION BY d1 ORDER BY (id1, d1); -CREATE TABLE test_03053.derived2 +CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.derived2 ( `id2` UInt64, `d2` UInt64, @@ -39,6 +39,6 @@ SELECT derived2.id2 AS `derived2.id2`, derived2.value2 AS `derived2.value2`, derived1.value1 AS `derived1.value1` -FROM test_03053.base AS base -LEFT JOIN test_03053.derived2 AS derived2 ON base.id2 = derived2.id2 -LEFT JOIN test_03053.derived1 AS derived1 ON base.id = derived1.id1; +FROM {CLICKHOUSE_DATABASE:Identifier}.base AS base +LEFT JOIN {CLICKHOUSE_DATABASE:Identifier}.derived2 AS derived2 ON base.id2 = derived2.id2 +LEFT JOIN {CLICKHOUSE_DATABASE:Identifier}.derived1 AS derived1 ON base.id = derived1.id1; diff --git a/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql b/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql index 03ad9c97d94..10d18324c3c 100644 --- a/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql +++ b/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql @@ -1,18 +1,18 @@ -- https://github.com/ClickHouse/ClickHouse/issues/61947 SET allow_experimental_analyzer=1; -DROP DATABASE IF EXISTS d3; -DROP DATABASE IF EXISTS d4; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier}; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; -CREATE DATABASE d3; -CREATE DATABASE d4; -CREATE TABLE d3.`1-1` (field Int8) ENGINE = Memory; -CREATE TABLE d4.`2-1` (field Int8) ENGINE = Memory; -CREATE TABLE d4.`3-1` (field Int8) ENGINE = Memory; +CREATE DATABASE {CLICKHOUSE_DATABASE:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.`1-1` (field Int8) ENGINE = Memory; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.`2-1` (field Int8) ENGINE = Memory; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.`3-1` (field Int8) ENGINE = Memory; -INSERT INTO d3.`1-1` VALUES (1); +INSERT INTO {CLICKHOUSE_DATABASE:Identifier}.`1-1` VALUES (1); -SELECT d3.`1-1`.* -FROM d3.`1-1` -LEFT JOIN d4.`2-1` ON d3.`1-1`.field = d4.`2-1`.field -LEFT JOIN d4.`3-1` ON d4.`2-1`.field = d4.`3-1`.field; +SELECT {CLICKHOUSE_DATABASE:Identifier}.`1-1`.* +FROM {CLICKHOUSE_DATABASE:Identifier}.`1-1` +LEFT JOIN {CLICKHOUSE_DATABASE_1:Identifier}.`2-1` ON {CLICKHOUSE_DATABASE:Identifier}.`1-1`.field = {CLICKHOUSE_DATABASE_1:Identifier}.`2-1`.field +LEFT JOIN {CLICKHOUSE_DATABASE_1:Identifier}.`3-1` ON {CLICKHOUSE_DATABASE_1:Identifier}.`2-1`.field = {CLICKHOUSE_DATABASE_1:Identifier}.`3-1`.field; From 6277deb166b5ca3451cc47b4a4abb71604505405 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:59:54 +0200 Subject: [PATCH 1095/1165] small fixes --- src/Storages/MemorySettings.cpp | 4 ++-- src/Storages/StorageMemory.cpp | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Storages/MemorySettings.cpp b/src/Storages/MemorySettings.cpp index 7c8c9da209b..30ae4e12668 100644 --- a/src/Storages/MemorySettings.cpp +++ b/src/Storages/MemorySettings.cpp @@ -46,14 +46,14 @@ void MemorySettings::sanityCheck() const { if (min_bytes_to_keep > max_bytes_to_keep) throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, - "`min_bytes_to_keep` setting cannot be higher than `max_bytes_to_keep`. `min_bytes_to_keep`: {}, `max_bytes_to_keep`: {}", + "Setting `min_bytes_to_keep` cannot be higher than the `max_bytes_to_keep`. `min_bytes_to_keep`: {}, `max_bytes_to_keep`: {}", min_bytes_to_keep, max_bytes_to_keep); if (min_rows_to_keep > max_rows_to_keep) throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, - "`min_rows_to_keep` setting cannot be higher than `max_rows_to_keep`. `min_rows_to_keep`: {}, `max_rows_to_keep`: {}", + "Setting `min_rows_to_keep` cannot be higher than the `max_rows_to_keep`. `min_rows_to_keep`: {}, `max_rows_to_keep`: {}", min_rows_to_keep, max_rows_to_keep); } diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index c5feb6dad21..f69c4adb552 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -304,14 +304,14 @@ void StorageMemory::alter(const DB::AlterCommands & params, DB::ContextPtr conte if (params.isSettingsAlter()) { auto & settings_changes = new_metadata.settings_changes->as(); - auto copy = memory_settings; - copy.applyChanges(settings_changes.changes); - copy.sanityCheck(); + auto changed_settings = memory_settings; + changed_settings.applyChanges(settings_changes.changes); + changed_settings.sanityCheck(); /// When modifying the values of max_bytes_to_keep and max_rows_to_keep to be smaller than the old values, /// the old data needs to be removed. - if (!memory_settings.max_bytes_to_keep || memory_settings.max_bytes_to_keep > copy.max_bytes_to_keep - || !memory_settings.max_rows_to_keep || memory_settings.max_rows_to_keep > copy.max_rows_to_keep) + if (!memory_settings.max_bytes_to_keep || memory_settings.max_bytes_to_keep > changed_settings.max_bytes_to_keep + || !memory_settings.max_rows_to_keep || memory_settings.max_rows_to_keep > changed_settings.max_rows_to_keep) { std::lock_guard lock(mutex); @@ -319,14 +319,14 @@ void StorageMemory::alter(const DB::AlterCommands & params, DB::ContextPtr conte UInt64 new_total_rows = total_size_rows.load(std::memory_order_relaxed); UInt64 new_total_bytes = total_size_bytes.load(std::memory_order_relaxed); while (!new_data->empty() - && ((copy.max_bytes_to_keep && new_total_bytes > copy.max_bytes_to_keep) - || (copy.max_rows_to_keep && new_total_rows > copy.max_rows_to_keep))) + && ((changed_settings.max_bytes_to_keep && new_total_bytes > changed_settings.max_bytes_to_keep) + || (changed_settings.max_rows_to_keep && new_total_rows > changed_settings.max_rows_to_keep))) { Block oldest_block = new_data->front(); UInt64 rows_to_remove = oldest_block.rows(); UInt64 bytes_to_remove = oldest_block.allocatedBytes(); - if (new_total_bytes - bytes_to_remove < copy.min_bytes_to_keep - || new_total_rows - rows_to_remove < copy.min_rows_to_keep) + if (new_total_bytes - bytes_to_remove < changed_settings.min_bytes_to_keep + || new_total_rows - rows_to_remove < changed_settings.min_rows_to_keep) { break; // stop - removing next block will put us under min_bytes / min_rows threshold } @@ -341,7 +341,7 @@ void StorageMemory::alter(const DB::AlterCommands & params, DB::ContextPtr conte total_size_rows.store(new_total_rows, std::memory_order_relaxed); total_size_bytes.store(new_total_bytes, std::memory_order_relaxed); } - memory_settings = std::move(copy); + memory_settings = std::move(changed_settings); } DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); From baa62cdaeeb23aba770efe6368bba6ec97cf6214 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Mon, 8 Apr 2024 16:09:47 +0000 Subject: [PATCH 1096/1165] CI: no CI Running status for MQ --- tests/ci/run_check.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 6187656983e..435a5f726f2 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -201,14 +201,17 @@ def main(): ci_report_url = create_ci_report(pr_info, []) print("::notice ::Can run") - post_commit_status( - commit, - PENDING, - ci_report_url, - description, - CI_STATUS_NAME, - pr_info, - ) + + if not pr_info.is_merge_queue(): + # we need clean CI status for MQ to merge (no pending statuses) + post_commit_status( + commit, + PENDING, + ci_report_url, + description, + CI_STATUS_NAME, + pr_info, + ) if __name__ == "__main__": From 39c6188a2c0b7014136e1d9d9f16c684741fb0cb Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 8 Apr 2024 16:38:19 +0000 Subject: [PATCH 1097/1165] Fix logical error 'numbers_storage.step != UInt64{0}' --- src/TableFunctions/TableFunctionNumbers.cpp | 4 ++++ .../03037_zero_step_in_numbers_table_function.reference | 0 .../0_stateless/03037_zero_step_in_numbers_table_function.sql | 2 ++ 3 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/03037_zero_step_in_numbers_table_function.reference create mode 100644 tests/queries/0_stateless/03037_zero_step_in_numbers_table_function.sql diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 2989eb5fbef..16f56eab981 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int BAD_ARGUMENTS; } namespace @@ -78,6 +79,9 @@ StoragePtr TableFunctionNumbers::executeImpl( UInt64 length = arguments.size() >= 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); UInt64 step = arguments.size() == 3 ? evaluateArgument(context, arguments[2]) : 1; + if (!step) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function {} requires step to be a positive number", getName()); + auto res = std::make_shared( StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset, step); res->startup(); diff --git a/tests/queries/0_stateless/03037_zero_step_in_numbers_table_function.reference b/tests/queries/0_stateless/03037_zero_step_in_numbers_table_function.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03037_zero_step_in_numbers_table_function.sql b/tests/queries/0_stateless/03037_zero_step_in_numbers_table_function.sql new file mode 100644 index 00000000000..08fafd6ddfa --- /dev/null +++ b/tests/queries/0_stateless/03037_zero_step_in_numbers_table_function.sql @@ -0,0 +1,2 @@ +select * from numbers(1, 10, 0); -- {serverError BAD_ARGUMENTS} + From e5282bf39fa1a930b0d93bb4077e1baf6082f6c7 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 8 Apr 2024 16:39:55 +0000 Subject: [PATCH 1098/1165] =?UTF-8?q?I=20=E2=99=A5=20group=5Fby=5Fuse=5Fnu?= =?UTF-8?q?lls.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 14 ++++++++++++-- .../03023_group_by_use_nulls_analyzer_crashes.sql | 8 ++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 4d862639e15..793cec91293 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -6694,8 +6694,11 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR { for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes()) { - for (const auto & group_by_elem : grouping_set->as()->getNodes()) + for (auto & group_by_elem : grouping_set->as()->getNodes()) + { + group_by_elem = group_by_elem->clone(); scope.nullable_group_by_keys.insert(group_by_elem); + } } } } @@ -6713,8 +6716,15 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR if (scope.group_by_use_nulls) { - for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) + for (auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) + { + /// Clone is needed cause aliases share subtrees. + /// If not clone, a part of GROUP BY key could be replaced to nullable + /// by replacing a part of alias from another subtree to nullable. + /// See 03023_group_by_use_nulls_analyzer_crashes + group_by_elem = group_by_elem->clone(); scope.nullable_group_by_keys.insert(group_by_elem); + } } } } diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql index 2f4c8b1c75e..b6ca454c1cf 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql @@ -9,3 +9,11 @@ SELECT tuple(number) AS x FROM numbers(10) GROUP BY GROUPING SETS (number) order SELECT ignore(toFixedString('Lambda as function parameter', 28), toNullable(28), ignore(8)), sum(marks) FROM system.parts WHERE database = currentDatabase() GROUP BY GROUPING SETS ((2)) FORMAT Null settings optimize_injective_functions_in_group_by=1, optimize_group_by_function_keys=1, group_by_use_nulls=1; -- { serverError ILLEGAL_AGGREGATION } SELECT toLowCardinality(materialize('a' AS key)), 'b' AS value GROUP BY key WITH CUBE SETTINGS group_by_use_nulls = 1; + +SELECT tuple(tuple(number)) AS x +FROM numbers(10) +GROUP BY (number, (toString(x), number)) + WITH CUBE +SETTINGS group_by_use_nulls = 1 FORMAT Null; + +SELECT tuple(number + 1) AS x FROM numbers(10) GROUP BY number + 1, toString(x) WITH CUBE settings group_by_use_nulls=1 FORMAT Null; From b318091528eed9db6d04d25bae115d24d3b82eb8 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 8 Apr 2024 17:17:04 +0000 Subject: [PATCH 1099/1165] Don't check overflow in dotProduct in undefined sanitizer --- src/Functions/array/arrayDotProduct.cpp | 4 ++-- .../queries/0_stateless/03037_dot_product_overflow.reference | 1 + tests/queries/0_stateless/03037_dot_product_overflow.sql | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03037_dot_product_overflow.reference create mode 100644 tests/queries/0_stateless/03037_dot_product_overflow.sql diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index 783843a89d5..4551140acc3 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -66,13 +66,13 @@ struct DotProduct }; template - static void accumulate(State & state, Type x, Type y) + static NO_SANITIZE_UNDEFINED void accumulate(State & state, Type x, Type y) { state.sum += x * y; } template - static void combine(State & state, const State & other_state) + static NO_SANITIZE_UNDEFINED void combine(State & state, const State & other_state) { state.sum += other_state.sum; } diff --git a/tests/queries/0_stateless/03037_dot_product_overflow.reference b/tests/queries/0_stateless/03037_dot_product_overflow.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/03037_dot_product_overflow.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/03037_dot_product_overflow.sql b/tests/queries/0_stateless/03037_dot_product_overflow.sql new file mode 100644 index 00000000000..94d5eba6255 --- /dev/null +++ b/tests/queries/0_stateless/03037_dot_product_overflow.sql @@ -0,0 +1,2 @@ +select ignore(dotProduct(materialize([9223372036854775807, 1]), materialize([-3, 1]))); + From 75aff7fc1aed1c56cfd406466a99d5739895f0c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 8 Apr 2024 15:50:10 +0200 Subject: [PATCH 1100/1165] Remove the code --- contrib/avro-cmake/CMakeLists.txt | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt index c99c7dd4624..96f740b6dd2 100644 --- a/contrib/avro-cmake/CMakeLists.txt +++ b/contrib/avro-cmake/CMakeLists.txt @@ -59,15 +59,3 @@ target_link_libraries (_avrocpp PRIVATE boost::headers_only boost::iostreams) target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR}) target_link_libraries (_avrocpp PRIVATE ch_contrib::snappy) - -# create a symlink to include headers with -set(AVRO_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include") -ADD_CUSTOM_COMMAND(OUTPUT "${AVRO_INCLUDE_DIR}" - COMMAND ${CMAKE_COMMAND} -E make_directory "${AVRO_INCLUDE_DIR}" - COMMAND ${CMAKE_COMMAND} -E create_symlink "${AVROCPP_ROOT_DIR}/api" "${AVRO_INCLUDE_DIR}/avro" - DEPENDS "${AVROCPP_ROOT_DIR}/api" -) -ADD_CUSTOM_TARGET(avro_symlink_headers ALL - DEPENDS "${AVRO_INCLUDE_DIR}") -add_dependencies(_avrocpp avro_symlink_headers) -target_include_directories(_avrocpp SYSTEM BEFORE PUBLIC "${AVRO_INCLUDE_DIR}") From 75e46f865c1d445c23d55e918f1bb83669250b46 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Apr 2024 19:19:18 +0200 Subject: [PATCH 1101/1165] Avoid uncaught exception for onFault handler onFault() is called from the std::thread, and it should catch all exceptions, otherwise you can unrelated fatal errors.
stacktrace 2024.04.08 13:15:29.526847 [ 2067427 ] {} BaseDaemon: (version 24.2.2.71 (official build), build id: 57F857DCFE8BA6838F6463E4665CD700852BFF0E, git hash: 9293d361e72be9f6ccfd444d504e2137b2e837cf) (from thread 2118603) Terminate called for uncaught exception: 2024.04.08 13:15:29.526904 [ 2067427 ] {} BaseDaemon: Code: 210. DB::NetException: I/O error: Broken pipe, while writing to socket (10.61.7.253:9000 -> 10.101.53.134:46036). (NETWORK_ERROR), Stack trace (when copying this message, always include the lines below): 2024.04.08 13:15:29.526983 [ 2067427 ] {} BaseDaemon: 2024.04.08 13:15:29.527042 [ 2067427 ] {} BaseDaemon: 0. ./build_docker/./src/Common/Exception.cpp:96: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000cf5af1b 2024.04.08 13:15:29.527061 [ 2067427 ] {} BaseDaemon: 1. ./contrib/llvm-project/libcxx/include/string:1499: DB::NetException::NetException(int, FormatStringHelperImpl::type, std::type_identity::type, std::type_identity::type>, String&&, String&&, String&&) @ 0x000000000d07dfe1 2024.04.08 13:15:29.527082 [ 2067427 ] {} BaseDaemon: 2. ./build_docker/./src/IO/WriteBufferFromPocoSocket.cpp:0: DB::WriteBufferFromPocoSocket::nextImpl() @ 0x000000000d07e97e 2024.04.08 13:15:29.527125 [ 2067427 ] {} BaseDaemon: 3. ./src/IO/WriteBuffer.h:65: DB::TCPHandler::sendLogs() @ 0x0000000012fd31c1 2024.04.08 13:15:29.527144 [ 2067427 ] {} BaseDaemon: 4. ./contrib/llvm-project/libcxx/include/atomic:958: void std::__function::__policy_invoker::__call_impl>(std::__function::__policy_storage const*) @ 0x0000000012fdcc6e 2024.04.08 13:15:29.527163 [ 2067427 ] {} BaseDaemon: 5. ./contrib/llvm-project/libcxx/include/__functional/function.h:0: ? @ 0x000000000d25c65b 2024.04.08 13:15:29.527182 [ 2067427 ] {} BaseDaemon: 6. ./build_docker/./src/Daemon/BaseDaemon.cpp:286: void* std::__thread_proxy[abi:v15000]>, SignalListener::run()::'lambda'()>>(void*) @ 0x000000000d25e775 2024.04.08 13:15:29.527191 [ 2067427 ] {} BaseDaemon: 7. ? @ 0x00007f0fe4906609 2024.04.08 13:15:29.527211 [ 2067427 ] {} BaseDaemon: 8. ? @ 0x00007f0fe482b353 2024.04.08 13:15:29.534235 [ 2118604 ] {} BaseDaemon: ########## Short fault info ############ 2024.04.08 13:15:29.534347 [ 2118604 ] {} BaseDaemon: (version 24.2.2.71 (official build), build id: 57F857DCFE8BA6838F6463E4665CD700852BFF0E, git hash: 9293d361e72be9f6ccfd444d504e2137b2e837cf) (from thread 2118603) Received signal 6 2024.04.08 13:15:29.534476 [ 2118604 ] {} BaseDaemon: Signal description: Aborted 2024.04.08 13:15:29.534484 [ 2118604 ] {} BaseDaemon: 2024.04.08 13:15:29.534510 [ 2118604 ] {} BaseDaemon: Stack trace: 0x00007f0fe474f00b 0x00007f0fe472e859 0x000000000d24f72e 0x0000000017d15be3 0x0000000017d15818 0x0000000012fd350d 0x0000000012fdcc6e 0x000000000d25c65b 0x000000000d25e775 0x00007f0fe4906609 0x00007f0fe482b353 2024.04.08 13:15:29.534531 [ 2118604 ] {} BaseDaemon: ######################################## 2024.04.08 13:15:29.534609 [ 2118604 ] {} BaseDaemon: (version 24.2.2.71 (official build), build id: 57F857DCFE8BA6838F6463E4665CD700852BFF0E, git hash: 9293d361e72be9f6ccfd444d504e2137b2e837cf) (from thread 2118603) (no query) Received signal Aborted (6) 2024.04.08 13:15:29.534638 [ 2118604 ] {} BaseDaemon: 2024.04.08 13:15:29.534663 [ 2118604 ] {} BaseDaemon: Stack trace: 0x00007f0fe474f00b 0x00007f0fe472e859 0x000000000d24f72e 0x0000000017d15be3 0x0000000017d15818 0x0000000012fd350d 0x0000000012fdcc6e 0x000000000d25c65b 0x000000000d25e775 0x00007f0fe4906609 0x00007f0fe482b353 2024.04.08 13:15:29.534711 [ 2118604 ] {} BaseDaemon: 2. ? @ 0x00007f0fe474f00b 2024.04.08 13:15:29.534728 [ 2118604 ] {} BaseDaemon: 3. ? @ 0x00007f0fe472e859 2024.04.08 13:15:29.613230 [ 2118604 ] {} BaseDaemon: 4.0. inlined from ./contrib/llvm-project/libcxxabi/src/cxa_exception.cpp:670: __cxa_decrement_exception_refcount 2024.04.08 13:15:29.613283 [ 2118604 ] {} BaseDaemon: 4.1. inlined from ./contrib/llvm-project/libcxx/src/support/runtime/exception_pointer_cxxabi.ipp:17: ~exception_ptr 2024.04.08 13:15:29.613311 [ 2118604 ] {} BaseDaemon: 4. ./build_docker/./src/Daemon/BaseDaemon.cpp:591: terminate_handler() @ 0x000000000d24f72e 2024.04.08 13:15:29.617590 [ 2118604 ] {} BaseDaemon: 5. ./build_docker/./contrib/llvm-project/libcxxabi/src/cxa_handlers.cpp:61: std::__terminate(void (*)()) @ 0x0000000017d15be3 2024.04.08 13:15:29.619575 [ 2118604 ] {} BaseDaemon: 6. ./build_docker/./contrib/llvm-project/libcxxabi/src/cxa_exception.cpp:0: __cxa_rethrow @ 0x0000000017d15818 2024.04.08 13:15:30.104097 [ 2118604 ] {} BaseDaemon: 7.0. inlined from ./src/IO/WriteBuffer.h:0: DB::WriteBuffer::next() 2024.04.08 13:15:30.104331 [ 2118604 ] {} BaseDaemon: 7.1. inlined from ./build_docker/./src/Server/TCPHandler.cpp:2225: DB::TCPHandler::sendLogData(DB::Block const&) 2024.04.08 13:15:30.104408 [ 2118604 ] {} BaseDaemon: 7. ./build_docker/./src/Server/TCPHandler.cpp:2303: DB::TCPHandler::sendLogs() @ 0x0000000012fd350d 2024.04.08 13:15:30.217481 [ 2118604 ] {} BaseDaemon: 8.0. inlined from ./contrib/llvm-project/libcxx/include/atomic:958: double std::__cxx_atomic_load[abi:v15000](std::__cxx_atomic_base_impl const*, std::memory_order) 2024.04.08 13:15:30.217579 [ 2118604 ] {} BaseDaemon: 8.1. inlined from ./contrib/llvm-project/libcxx/include/atomic:1588: std::__atomic_base::load[abi:v15000](std::memory_order) const 2024.04.08 13:15:30.217617 [ 2118604 ] {} BaseDaemon: 8.2. inlined from ./src/Common/ThreadFuzzer.cpp:407: pthread_mutex_unlock 2024.04.08 13:15:30.217644 [ 2118604 ] {} BaseDaemon: 8.3. inlined from ./contrib/llvm-project/libcxx/include/__threading_support:314: std::__libcpp_mutex_unlock[abi:v15000](pthread_mutex_t*) 2024.04.08 13:15:30.217676 [ 2118604 ] {} BaseDaemon: 8.4. inlined from ./contrib/llvm-project/libcxx/src/mutex.cpp:52: std::mutex::unlock() 2024.04.08 13:15:30.217699 [ 2118604 ] {} BaseDaemon: 8.5. inlined from ./contrib/llvm-project/libcxx/include/__mutex_base:100: ~lock_guard 2024.04.08 13:15:30.217747 [ 2118604 ] {} BaseDaemon: 8.6. inlined from ./build_docker/./src/Server/TCPHandler.cpp:392: operator() 2024.04.08 13:15:30.217776 [ 2118604 ] {} BaseDaemon: 8.7. inlined from ./contrib/llvm-project/libcxx/include/__functional/invoke.h:394: ? 2024.04.08 13:15:30.217796 [ 2118604 ] {} BaseDaemon: 8.8. inlined from ./contrib/llvm-project/libcxx/include/__functional/invoke.h:479: ? 2024.04.08 13:15:30.217859 [ 2118604 ] {} BaseDaemon: 8.9. inlined from ./contrib/llvm-project/libcxx/include/__functional/function.h:235: ? 2024.04.08 13:15:30.217878 [ 2118604 ] {} BaseDaemon: 8. ./contrib/llvm-project/libcxx/include/__functional/function.h:716: ? @ 0x0000000012fdcc6e 2024.04.08 13:15:30.240809 [ 2118604 ] {} BaseDaemon: 9. ./contrib/llvm-project/libcxx/include/__functional/function.h:0: ? @ 0x000000000d25c65b 2024.04.08 13:15:30.283617 [ 2118604 ] {} BaseDaemon: 10.0. inlined from ./build_docker/./src/Daemon/BaseDaemon.cpp:286: operator() 2024.04.08 13:15:30.283686 [ 2118604 ] {} BaseDaemon: 10.1. inlined from ./contrib/llvm-project/libcxx/include/__functional/invoke.h:394: ? 2024.04.08 13:15:30.283725 [ 2118604 ] {} BaseDaemon: 10.2. inlined from ./contrib/llvm-project/libcxx/include/thread:284: void std::__thread_execute[abi:v15000]>, SignalListener::run()::'lambda'()>(std::tuple>, SignalListener::run()::'lambda'()>&, std::__tuple_indices<>) 2024.04.08 13:15:30.283755 [ 2118604 ] {} BaseDaemon: 10. ./contrib/llvm-project/libcxx/include/thread:295: void* std::__thread_proxy[abi:v15000]>, SignalListener::run()::'lambda'()>>(void*) @ 0x000000000d25e775 2024.04.08 13:15:30.283799 [ 2118604 ] {} BaseDaemon: 11. ? @ 0x00007f0fe4906609 2024.04.08 13:15:30.283821 [ 2118604 ] {} BaseDaemon: 12. ? @ 0x00007f0fe482b353 2024.04.08 13:15:30.574588 [ 2118604 ] {} BaseDaemon: Integrity check of the executable successfully passed (checksum: 3485110FABDB0C94202BD684999A9814) 2024.04.08 13:15:30.574704 [ 2118604 ] {} BaseDaemon: Report this error to https://github.com/ClickHouse/ClickHouse/issues
v2: fatal logging Signed-off-by: Azat Khuzhin --- src/Daemon/BaseDaemon.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index cc22db3969c..019ad8e716b 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -332,6 +332,7 @@ private: const std::vector & thread_frame_pointers, UInt32 thread_num, ThreadStatus * thread_ptr) const + try { ThreadStatus thread_status; @@ -543,6 +544,11 @@ private: fatal_error_printed.test_and_set(); } + catch (...) + { + PreformattedMessage message = getCurrentExceptionMessageAndPattern(true); + LOG_FATAL(getLogger(__PRETTY_FUNCTION__), message); + } }; From 5db0df0bc1ff7be9e39dac189c7bfe2dfb6179ae Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Apr 2024 21:48:36 +0300 Subject: [PATCH 1102/1165] Update BaseDaemon.cpp --- src/Daemon/BaseDaemon.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 019ad8e716b..592ca4e55d5 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -520,7 +520,7 @@ private: } } - /// ClickHouse Keeper does not link to some part of Settings. + /// ClickHouse Keeper does not link to some parts of Settings. #ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD /// List changed settings. if (!query_id.empty()) @@ -538,7 +538,7 @@ private: } #endif - /// When everything is done, we will try to send these error messages to client. + /// When everything is done, we will try to send these error messages to the client. if (thread_ptr) thread_ptr->onFatalError(); @@ -546,6 +546,7 @@ private: } catch (...) { + /// onFault is called from the std::thread, and it should catch all exceptions; otherwise, you can get unrelated fatal errors. PreformattedMessage message = getCurrentExceptionMessageAndPattern(true); LOG_FATAL(getLogger(__PRETTY_FUNCTION__), message); } From 33eba42c13c6894b05972df9588b49a78051d5f0 Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 8 Apr 2024 20:50:41 +0200 Subject: [PATCH 1103/1165] Document wyHash64 --- .../sql-reference/functions/hash-functions.md | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 90c7d8c2206..ab97e43c624 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -1077,9 +1077,7 @@ Result: ## wordShingleSimHashUTF8 -Splits a UTF-8 string into parts (shingles) of `shinglesize` words and returns the word shingle `simhash`. Is case sensitive. -Can be used for detection of semi-duplicate strings with [bitHammingDistance](/docs/en/sql-reference/functions/bit-functions.md/#bithammingdistance). The smaller is the [Hamming Distance](https://en.wikipedia.org/wiki/Hamming_distance) of the calculated `simhashes` of two strings, the more likely these strings are the same. **Syntax** @@ -1153,6 +1151,42 @@ Result: └────────────┘ ``` +## wyHash64 + +Produces a 64-bit [wyHash64](https://github.com/wangyi-fudan/wyhash) hash value. + +**Syntax** + +```sql +wyHash64(string) +``` + +**Arguments** + +- `string` — String. [String](/docs/en/sql-reference/data-types/string.md). + +**Returned value** + +- Hash value. + +Type: [UInt64](/docs/en/sql-reference/data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT wyHash64('ClickHouse') AS Hash; +``` + +Result: + +```response +┌─────────────────Hash─┐ +│ 12336419557878201794 │ +└──────────────────────┘ +``` + ## ngramMinHash Splits a ASCII string into n-grams of `ngramsize` symbols and calculates hash values for each n-gram. Uses `hashnum` minimum hashes to calculate the minimum hash and `hashnum` maximum hashes to calculate the maximum hash. Returns a tuple with these hashes. Is case sensitive. From 9a08f1ddd6f414097a6bf00f1159924d4a14b4e4 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 8 Apr 2024 20:53:19 +0200 Subject: [PATCH 1104/1165] Update StorageFileLog.cpp --- src/Storages/FileLog/StorageFileLog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 7b0cfdf6a6c..ba6528b6e4c 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -152,7 +152,7 @@ StorageFileLog::StorageFileLog( if (!fileOrSymlinkPathStartsWith(path, getContext()->getUserFilesPath())) { - if (LoadingStrictnessLevel::ATTACH <= mode) + if (LoadingStrictnessLevel::SECONDARY_CREATE <= mode) { LOG_ERROR(log, "The absolute data path should be inside `user_files_path`({})", getContext()->getUserFilesPath()); return; From 3f91ece54b140906f050c0d6fc11563b81e48364 Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 8 Apr 2024 20:54:42 +0200 Subject: [PATCH 1105/1165] Remove blank space from wordShingleSimHashUTF8 --- docs/en/sql-reference/functions/hash-functions.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index ab97e43c624..9bfaaae5463 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -1077,8 +1077,6 @@ Result: ## wordShingleSimHashUTF8 - - **Syntax** ```sql From 1b9a72e374f0194d3ab856f5cc0cd491a65af9a4 Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 8 Apr 2024 20:56:30 +0200 Subject: [PATCH 1106/1165] Add back accidently removed description --- docs/en/sql-reference/functions/hash-functions.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 9bfaaae5463..902e5ab9baf 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -1077,6 +1077,10 @@ Result: ## wordShingleSimHashUTF8 +Splits a UTF-8 string into parts (shingles) of `shinglesize` words and returns the word shingle `simhash`. Is case sensitive. + +Can be used for detection of semi-duplicate strings with [bitHammingDistance](/docs/en/sql-reference/functions/bit-functions.md/#bithammingdistance). The smaller is the [Hamming Distance](https://en.wikipedia.org/wiki/Hamming_distance) of the calculated `simhashes` of two strings, the more likely these strings are the same. + **Syntax** ```sql From 2280fdeec1d41fdeb7a09459577312de8dc70bec Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 8 Apr 2024 19:16:47 +0000 Subject: [PATCH 1107/1165] Empty commit From b138b1e103d6ccba62620b849931a9a607e9a42b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 8 Apr 2024 19:18:44 +0000 Subject: [PATCH 1108/1165] Empty commit From 208722a5133caac4f8d1e54afd0d301a1ff0d2de Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 8 Apr 2024 21:55:27 +0200 Subject: [PATCH 1109/1165] Fix spelling mistake --- docs/en/sql-reference/functions/string-search-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 8aff8b7e930..9b457880991 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -13,7 +13,7 @@ Case-insensitive search follows the lowercase-uppercase rules of the English lan `I` whereas in the Turkish language it is `İ` - results for languages other than English may be unexpected. ::: -Functions in this section also assume that the searched string (refered to in this section as `haystack`) and the search string (refered to in this section as `needle`) are single-byte encoded text. If this assumption is +Functions in this section also assume that the searched string (referred to in this section as `haystack`) and the search string (referred to in this section as `needle`) are single-byte encoded text. If this assumption is violated, no exception is thrown and results are undefined. Search with UTF-8 encoded strings is usually provided by separate function variants. Likewise, if a UTF-8 function variant is used and the input strings are not UTF-8 encoded text, no exception is thrown and the results are undefined. Note that no automatic Unicode normalization is performed, however you can use the From 2c41bcb25aae59f4f7964ef112e0a9c426c65f27 Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 8 Apr 2024 21:57:11 +0200 Subject: [PATCH 1110/1165] Add multiSearchXYZ functions to aspell-dict.txt --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 57a8e0d5840..ebc53845011 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -550,6 +550,17 @@ Mongodb mortonDecode mortonEncode MsgPack +multiSearchAllPositionsCaseInsensitive +multiSearchAllPositionsCaseInsensitiveUTF +multiSearchAnyCaseInsensitive +multiSearchAnyCaseInsensitiveUTF +multiSearchAnyUTF +multiSearchFirstIndexCaseInsensitive +multiSearchFirstIndexCaseInsensitiveUTF +multiSearchFirstIndexUTF +multiSearchFirstPositionCaseInsensitive +multiSearchFirstPositionCaseInsensitiveUTF +multiSearchFirstPositionUTF MultiPolygon Multiline Multiqueries From f20771542633e85b07f49674e861740f44d1fe3e Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 8 Apr 2024 22:25:50 +0200 Subject: [PATCH 1111/1165] Update 03068_analyzer_distributed_join.sql --- tests/queries/0_stateless/03068_analyzer_distributed_join.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03068_analyzer_distributed_join.sql b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql index 61b1199dc44..542380feb7c 100644 --- a/tests/queries/0_stateless/03068_analyzer_distributed_join.sql +++ b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql @@ -1,5 +1,5 @@ --- https://github.com/ClickHouse/ClickHouse/issues/6571 --- Tag: no-replicated-database +-- Tags: no-replicated-database +-- Closes: https://github.com/ClickHouse/ClickHouse/issues/6571 SET allow_experimental_analyzer=1; CREATE TABLE LINEITEM_shard ON CLUSTER test_shard_localhost From eac8852c75da47991fa85c0611e8ef53c7059474 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 9 Apr 2024 00:14:49 +0200 Subject: [PATCH 1112/1165] Less flaky tests --- ...8_analyzer_ambiguous_column_multi_call.sql | 12 +++---- ...same_table_name_in_different_databases.sql | 32 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql index 4ca5005fa1d..e6f1ed81f91 100644 --- a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql +++ b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql @@ -1,13 +1,13 @@ -- https://github.com/ClickHouse/ClickHouse/issues/61014 SET allow_experimental_analyzer=1; -DROP DATABASE IF EXISTS test_03088; -create database test_03088; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier}; +create database {CLICKHOUSE_DATABASE:Identifier}; -create table test_03088.a (i int) engine = Log(); +create table {CLICKHOUSE_DATABASE:Identifier}.a (i int) engine = Log(); select - test_03088.a.i + {CLICKHOUSE_DATABASE:Identifier}.a.i from - test_03088.a, - test_03088.a as x; + {CLICKHOUSE_DATABASE:Identifier}.a, + {CLICKHOUSE_DATABASE:Identifier}.a as x; diff --git a/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql index 2185b5f450a..436f9395fc4 100644 --- a/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql +++ b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql @@ -1,28 +1,28 @@ -- https://github.com/ClickHouse/ClickHouse/issues/61947 SET allow_experimental_analyzer=1; -DROP DATABASE IF EXISTS d1; -DROP DATABASE IF EXISTS d2; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier}; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; -CREATE DATABASE d1; -CREATE DATABASE d2; -CREATE TABLE d1.`1-1` (field Int8) ENGINE = Memory; -CREATE TABLE d2.`1-1` (field Int8) ENGINE = Memory; -CREATE TABLE d2.`2-1` (field Int8) ENGINE = Memory; +CREATE DATABASE {CLICKHOUSE_DATABASE:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.`1-1` (field Int8) ENGINE = Memory; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.`1-1` (field Int8) ENGINE = Memory; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.`2-1` (field Int8) ENGINE = Memory; -INSERT INTO d1.`1-1` VALUES (1); +INSERT INTO {CLICKHOUSE_DATABASE:Identifier}.`1-1` VALUES (1); SELECT * -FROM d1.`1-1` -LEFT JOIN d2.`1-1` ON d1.`1-1`.field = d2.`1-1`.field; +FROM {CLICKHOUSE_DATABASE:Identifier}.`1-1` +LEFT JOIN {CLICKHOUSE_DATABASE_1:Identifier}.`1-1` ON {CLICKHOUSE_DATABASE:Identifier}.`1-1`.field = {CLICKHOUSE_DATABASE_1:Identifier}.`1-1`.field; SELECT ''; -SELECT 'using asterisk', d1.`1-1`.*, d2.`1-1`.* -FROM d1.`1-1` -LEFT JOIN d2.`1-1` USING field +SELECT 'using asterisk', {CLICKHOUSE_DATABASE:Identifier}.`1-1`.*, {CLICKHOUSE_DATABASE_1:Identifier}.`1-1`.* +FROM {CLICKHOUSE_DATABASE:Identifier}.`1-1` +LEFT JOIN {CLICKHOUSE_DATABASE_1:Identifier}.`1-1` USING field UNION ALL -SELECT 'using field name', d1.`1-1`.field, d2.`1-1`.field -FROM d1.`1-1` -LEFT JOIN d2.`1-1` USING field +SELECT 'using field name', {CLICKHOUSE_DATABASE:Identifier}.`1-1`.field, {CLICKHOUSE_DATABASE_1:Identifier}.`1-1`.field +FROM {CLICKHOUSE_DATABASE:Identifier}.`1-1` +LEFT JOIN {CLICKHOUSE_DATABASE_1:Identifier}.`1-1` USING field ORDER BY *; From f45acef1316dd1f15becd386e880471642765f44 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Tue, 9 Apr 2024 00:33:18 +0000 Subject: [PATCH 1113/1165] Support for a tiny feature in stateless tests image --- docker/test/stateless/run.sh | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index bac9d8df7a9..24b821f1d7d 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -41,6 +41,8 @@ source /utils.lib if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then echo "Azure is disabled" +elif [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + echo "Azure is disabled" else azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & fi @@ -137,6 +139,32 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited) fi +if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \ + | sed "s|/var/lib/clickhouse/filesystem_caches/|/var/lib/clickhouse/filesystem_caches_1/|" \ + > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp + mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml + + sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \ + | sed "s|/var/lib/clickhouse/filesystem_caches/|/var/lib/clickhouse/filesystem_caches_1/|" \ + > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp + mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml + + mkdir -p /var/run/clickhouse-server1 + sudo chown clickhouse:clickhouse /var/run/clickhouse-server1 + sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \ + --pid-file /var/run/clickhouse-server1/clickhouse-server.pid \ + -- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \ + --logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \ + --tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \ + --mysql_port 19004 --postgresql_port 19005 \ + --keeper_server.tcp_port 19181 --keeper_server.server_id 2 \ + --prometheus.port 19988 \ + --macros.replica r2 # It doesn't work :( + + MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000)) # min(MAX_RUN_TIME, 2.5 hours) + MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited) +fi # Wait for the server to start, but not for too long. for _ in {1..100} @@ -183,6 +211,10 @@ function run_tests() ADDITIONAL_OPTIONS+=('--s3-storage') fi + if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + ADDITIONAL_OPTIONS+=('--shared-catalog') + fi + if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then ADDITIONAL_OPTIONS+=('--replicated-database') # Too many tests fail for DatabaseReplicated in parallel. @@ -264,6 +296,12 @@ do echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 fi + + if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) + echo "$err" + [[ "0" != "${#err}" ]] && failed_to_save_logs=1 + fi done # Stop server so we can safely read data with clickhouse-local. @@ -275,6 +313,10 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||: fi +if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||: +fi + rg -Fa "" /var/log/clickhouse-server/clickhouse-server.log ||: rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||: zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst & @@ -302,6 +344,10 @@ if [ $failed_to_save_logs -ne 0 ]; then clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||: clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst ||: fi + + if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||: + fi done fi @@ -341,3 +387,10 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||: tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||: fi + +if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + rg -Fa "" /var/log/clickhouse-server/clickhouse-server1.log ||: + zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||: + mv /var/log/clickhouse-server/stderr1.log /test_output/ ||: + tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||: +fi From 0265ba502716d8536c85ea8914ba791f1d278c66 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 10:55:08 +0200 Subject: [PATCH 1114/1165] Add wyHash to aspell-ignore --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 8aa2a463c47..84c70d91879 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -2760,6 +2760,7 @@ wordShingleSimHashUTF wordshingleMinHash writability wrt +wyHash xcode xeus xkcd From 6e90a197f8e88a2d0a74490e96455bc203365295 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 9 Apr 2024 12:02:11 +0200 Subject: [PATCH 1115/1165] Update 03091_analyzer_same_table_name_in_different_databases.sql --- ...03091_analyzer_same_table_name_in_different_databases.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql index 436f9395fc4..599275c66e8 100644 --- a/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql +++ b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql @@ -18,6 +18,8 @@ LEFT JOIN {CLICKHOUSE_DATABASE_1:Identifier}.`1-1` ON {CLICKHOUSE_DATABASE:Ident SELECT ''; +SELECT * FROM +( SELECT 'using asterisk', {CLICKHOUSE_DATABASE:Identifier}.`1-1`.*, {CLICKHOUSE_DATABASE_1:Identifier}.`1-1`.* FROM {CLICKHOUSE_DATABASE:Identifier}.`1-1` LEFT JOIN {CLICKHOUSE_DATABASE_1:Identifier}.`1-1` USING field @@ -25,4 +27,5 @@ UNION ALL SELECT 'using field name', {CLICKHOUSE_DATABASE:Identifier}.`1-1`.field, {CLICKHOUSE_DATABASE_1:Identifier}.`1-1`.field FROM {CLICKHOUSE_DATABASE:Identifier}.`1-1` LEFT JOIN {CLICKHOUSE_DATABASE_1:Identifier}.`1-1` USING field -ORDER BY *; +) +ORDER BY ALL; From 0543fc3263dba3cf56e1445725f3589e43d91b21 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 9 Apr 2024 13:17:42 +0300 Subject: [PATCH 1116/1165] OptimizeGroupByInjectiveFunctionsPass remove unused constant --- .../OptimizeGroupByInjectiveFunctionsPass.cpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp index 61893202525..a30ad2a1590 100644 --- a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp @@ -12,24 +12,6 @@ namespace DB namespace { -const std::unordered_set possibly_injective_function_names -{ - "dictGet", - "dictGetString", - "dictGetUInt8", - "dictGetUInt16", - "dictGetUInt32", - "dictGetUInt64", - "dictGetInt8", - "dictGetInt16", - "dictGetInt32", - "dictGetInt64", - "dictGetFloat32", - "dictGetFloat64", - "dictGetDate", - "dictGetDateTime" -}; - class OptimizeGroupByInjectiveFunctionsVisitor : public InDepthQueryTreeVisitorWithContext { using Base = InDepthQueryTreeVisitorWithContext; From 4e344f6a9397d27a15af82023b182469eaeebd35 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Tue, 9 Apr 2024 10:25:41 +0000 Subject: [PATCH 1117/1165] remove ci status and reports for MQ case --- tests/ci/commit_status_helper.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index bda2db13991..56728c3d3ba 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -148,6 +148,11 @@ def set_status_comment(commit: Commit, pr_info: PRInfo) -> None: """It adds or updates the comment status to all Pull Requests but for release one, so the method does nothing for simple pushes and pull requests with `release`/`release-lts` labels""" + + if pr_info.is_merge_queue(): + # skip report creation for the MQ + return + # to reduce number of parameters, the Github is constructed on the fly gh = Github() gh.__requester = commit._requester # type:ignore #pylint:disable=protected-access @@ -441,7 +446,9 @@ def update_mergeable_check(commit: Commit, pr_info: PRInfo, check_name: str) -> or pr_info.release_pr or pr_info.number == 0 ) - if not_run: + + # FIXME: For now, always set mergeable check in the Merge Queue. It's required to pass MQ + if not_run and not pr_info.is_merge_queue(): # Let's avoid unnecessary work return From fe868ddf86ccaa60bccbe44afbfbe1e24ecf6c5d Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 12:31:35 +0200 Subject: [PATCH 1118/1165] Document uniqCombined64 and update uniqueCombined --- .../reference/uniqcombined.md | 19 +++-- .../reference/uniqcombined64.md | 76 ++++++++++++++++++- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md index 2f3efde859d..99a46c14a30 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md +++ b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md @@ -15,9 +15,9 @@ The `uniqCombined` function is a good choice for calculating the number of diffe **Arguments** -The function takes a variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types. +- `HLL_precision`: The base-2 logarithm of the number of cells in [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Optional, you can use the function as `uniqCombined(x[, ...])`. The default value for `HLL_precision` is 17, which is effectively 96 KiB of space (2^17 cells, 6 bits each). +- `X`: A variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types. -`HLL_precision` is the base-2 logarithm of the number of cells in [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Optional, you can use the function as `uniqCombined(x[, ...])`. The default value for `HLL_precision` is 17, which is effectively 96 KiB of space (2^17 cells, 6 bits each). **Returned value** @@ -25,26 +25,29 @@ The function takes a variable number of parameters. Parameters can be `Tuple`, ` **Implementation details** -Function: +The `uniqCombined` function: - Calculates a hash (64-bit hash for `String` and 32-bit otherwise) for all parameters in the aggregate, then uses it in calculations. - - Uses a combination of three algorithms: array, hash table, and HyperLogLog with an error correction table. - - For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. - + - For a small number of distinct elements, an array is used. + - When the set size is larger, a hash table is used. + - For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. - Provides the result deterministically (it does not depend on the query processing order). :::note Since it uses 32-bit hash for non-`String` type, the result will have very high error for cardinalities significantly larger than `UINT_MAX` (error will raise quickly after a few tens of billions of distinct values), hence in this case you should use [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64) ::: -Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined`: +Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined` function: - Consumes several times less memory. - Calculates with several times higher accuracy. - Usually has slightly lower performance. In some scenarios, `uniqCombined` can perform better than `uniq`, for example, with distributed queries that transmit a large number of aggregation states over the network. +**Example** + + + **See Also** - [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md index 9f010da57f2..b6e09bcaae3 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md +++ b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md @@ -5,4 +5,78 @@ sidebar_position: 193 # uniqCombined64 -Same as [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined), but uses 64-bit hash for all data types. +Calculates the approximate number of different argument values. It is the same as [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined), but uses a 64-bit hash for all data types rather than just for the String data type. + +``` sql +uniqCombined64(HLL_precision)(x[, ...]) +``` + +**Parameters** + +- `HLL_precision`: The base-2 logarithm of the number of cells in [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Optionally, you can use the function as `uniqCombined64(x[, ...])`. The default value for `HLL_precision` is 17, which is effectively 96 KiB of space (2^17 cells, 6 bits each). +- `X`: A variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types. + +**Returned value** + +- A number [UInt64](../../../sql-reference/data-types/int-uint.md)-type number. + +**Implementation details** + +The `uniqCombined64` function: +- Calculates a hash (64-bit hash for all data types) for all parameters in the aggregate, then uses it in calculations. +- Uses a combination of three algorithms: array, hash table, and HyperLogLog with an error correction table. + - For a small number of distinct elements, an array is used. + - When the set size is larger, a hash table is used. + - For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. +- Provides the result deterministically (it does not depend on the query processing order). + +:::note +Since it uses 64-bit hash for all types, the result does not suffer from very high error for cardinalities significantly larger than `UINT_MAX` like [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md) does, which uses a 32-bit hash for non-`String` types. +::: + +Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined64` function: + +- Consumes several times less memory. +- Calculates with several times higher accuracy. + +**Example** + +In the example below `uniqCombined64` is run on `1e10` different numbers returning a very close approximation of the number of different argument values. + +Query: + +```sql +SELECT uniqCombined64(number) FROM numbers(1e10); +``` + +Result: + +```response +┌─uniqCombined64(number)─┐ +│ 9998568925 │ -- 10.00 billion +└────────────────────────┘ +``` + +By comparison the `uniqCombined` function returns a rather poor approximation for an input this size. + +Query: + +```sql +SELECT uniqCombined(number) FROM numbers(1e10); +``` + +Result: + +```response +┌─uniqCombined(number)─┐ +│ 5545308725 │ -- 5.55 billion +└──────────────────────┘ +``` + +**See Also** + +- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) +- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md) +- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12) +- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact) +- [uniqTheta](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch) From af505aafbca25335b3f2a5659e681af373189231 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 12:34:53 +0200 Subject: [PATCH 1119/1165] Small fix --- .../aggregate-functions/reference/uniqcombined.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md index 99a46c14a30..4d47aa6621e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md +++ b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md @@ -29,13 +29,13 @@ The `uniqCombined` function: - Calculates a hash (64-bit hash for `String` and 32-bit otherwise) for all parameters in the aggregate, then uses it in calculations. - Uses a combination of three algorithms: array, hash table, and HyperLogLog with an error correction table. - - For a small number of distinct elements, an array is used. - - When the set size is larger, a hash table is used. - - For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. + - For a small number of distinct elements, an array is used. + - When the set size is larger, a hash table is used. + - For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. - Provides the result deterministically (it does not depend on the query processing order). :::note -Since it uses 32-bit hash for non-`String` type, the result will have very high error for cardinalities significantly larger than `UINT_MAX` (error will raise quickly after a few tens of billions of distinct values), hence in this case you should use [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64) +Since it uses a 32-bit hash for non-`String` types, the result will have very high error for cardinalities significantly larger than `UINT_MAX` (error will raise quickly after a few tens of billions of distinct values), hence in this case you should use [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64). ::: Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined` function: From 7b3a973ee07d55ea6f8fe2cf6dfe2475e8cefc35 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 12:55:01 +0200 Subject: [PATCH 1120/1165] Add missing example to uniqCombined --- .../aggregate-functions/reference/uniqcombined.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md index 4d47aa6621e..18f44d2fcc4 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md +++ b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md @@ -46,7 +46,21 @@ Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq **Example** +Query: +```sql +SELECT uniqCombined(number) FROM numbers(1e6); +``` + +Result: + +```response +┌─uniqCombined(number)─┐ +│ 1001148 │ -- 1.00 million +└──────────────────────┘ +``` + +See the example section of [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64) for an example of the difference between `uniqCombined` and `uniqCombined64` for much larger inputs. **See Also** From 35e1e5aff7df25401ab3db59399f5bd49ba0deea Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 9 Apr 2024 14:42:29 +0300 Subject: [PATCH 1121/1165] Perf script update path in documentation --- tests/performance/scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/scripts/README.md b/tests/performance/scripts/README.md index 0a0580c62a0..1a15189fe86 100644 --- a/tests/performance/scripts/README.md +++ b/tests/performance/scripts/README.md @@ -130,7 +130,7 @@ More stages are available, e.g. restart servers or run the tests. See the code. #### Run a single test on the already configured servers ``` -docker/test/performance-comparison/perf.py --host=localhost --port=9000 --runs=1 tests/performance/logical_functions_small.xml +tests/performance/scripts/perf.py --host=localhost --port=9000 --runs=1 tests/performance/logical_functions_small.xml ``` #### Run all tests on some custom configuration From 55798dbdcf4222031afbf945f099d5f3bc111659 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 13:53:31 +0200 Subject: [PATCH 1122/1165] Add tupleIntDiv function --- .../functions/tuple-functions.md | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index b089de67e98..873065d226b 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -584,6 +584,59 @@ SELECT tupleConcat((1, 2), (3, 4), (true, false)) AS res └──────────────────────┘ ``` +## tupleIntDiv + +Does integer division of two tuples. + +**Syntax** + +```sql +tupleIntDiv(tuple1, tuple2) +``` + +**Parameters** + +- `tuple1`: tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `tuple2`: tuple of divisor values. [Tuple](../data-types/tuple) of numeric type. + +**Returned value** + +- Tuple of the quotients of `tuple1` and `tuple2`. [Tuple](../data-types/tuple) of integer values. + +**Implementation details** + +- If either `tuple1` or `tuple2` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor. + +**Examples** + +Query: + +``` sql +SELECT tupleIntDiv((15, 10, 5),(5, 5, 5)); +``` + +Result: + +``` text +┌─tupleIntDiv((15, 10, 5), (5, 5, 5))─┐ +│ (3,2,1) │ +└─────────────────────────────────────┘ +``` + +Query: + +``` sql +SELECT tupleIntDiv((15, 10, 5),(5.5, 5.5, 5.5)); +``` + +Result: + +``` text +┌─tupleIntDiv((15, 10, 5), (5.5, 5.5, 5.5))─┐ +│ (2,1,0) │ +└───────────────────────────────────────────┘ +``` + ## Distance functions All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md). From eed1a33e775599f7fa14dfdcdcb49f41939a9c1d Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 13:55:10 +0200 Subject: [PATCH 1123/1165] update wording on tupleIntDiv --- docs/en/sql-reference/functions/tuple-functions.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 873065d226b..7b626f8ab62 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -586,7 +586,7 @@ SELECT tupleConcat((1, 2), (3, 4), (true, false)) AS res ## tupleIntDiv -Does integer division of two tuples. +Does integer division of two tuples, returns a tuple of the quotients. **Syntax** @@ -637,6 +637,9 @@ Result: └───────────────────────────────────────────┘ ``` +## tupleIntDivByNumber + + ## Distance functions All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md). From 5857ecf8481b60e101c1550a073a2176c59b8f61 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 14:01:50 +0200 Subject: [PATCH 1124/1165] Add tupleIntDivByNumber --- .../functions/tuple-functions.md | 62 +++++++++++++++++-- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 7b626f8ab62..58613c6194b 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -586,26 +586,26 @@ SELECT tupleConcat((1, 2), (3, 4), (true, false)) AS res ## tupleIntDiv -Does integer division of two tuples, returns a tuple of the quotients. +Does integer division of a tuple of numerators and a tuple of denominators, and returns a tuple of the quotients. **Syntax** ```sql -tupleIntDiv(tuple1, tuple2) +tupleIntDiv(tuple_num, tuple_div) ``` **Parameters** -- `tuple1`: tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. -- `tuple2`: tuple of divisor values. [Tuple](../data-types/tuple) of numeric type. +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `tuple_div`: Tuple of divisor values. [Tuple](../data-types/tuple) of numeric type. **Returned value** -- Tuple of the quotients of `tuple1` and `tuple2`. [Tuple](../data-types/tuple) of integer values. +- Tuple of the quotients of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of integer values. **Implementation details** -- If either `tuple1` or `tuple2` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor. +- If either `tuple_num` or `tuple_div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor. **Examples** @@ -639,6 +639,56 @@ Result: ## tupleIntDivByNumber +Does integer division of a tuple of numerators by a given denominator, and returns a tuple of the quotients. + +**Syntax** + +```sql +tupleIntDivByNumber(tuple_num, div) +``` + +**Parameters** + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `div`: The divisor value. [Tuple](../data-types/tuple) of numeric type. + +**Returned value** + +- Tuple of the quotients of `tuple_num` and `div`. [Tuple](../data-types/tuple) of integer values. + +**Implementation details** + +- If either `tuple_num` or `div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor. + +**Examples** + +Query: + +``` sql +SELECT tupleIntDivByNumber((15, 10, 5),5); +``` + +Result: + +``` text +┌─tupleIntDivByNumber((15, 10, 5), 5)─┐ +│ (3,2,1) │ +└─────────────────────────────────────┘ +``` + +Query: + +``` sql +SELECT tupleIntDivByNumber((15.2, 10.7, 5.5),5.8); +``` + +Result: + +``` text +┌─tupleIntDivByNumber((15.2, 10.7, 5.5), 5.8)─┐ +│ (2,1,0) │ +└─────────────────────────────────────────────┘ +``` ## Distance functions From c8fb88b261b4bf1bbab57fb82bf895d471308126 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 14:15:32 +0200 Subject: [PATCH 1125/1165] Add tupleIntDivOrZero --- .../functions/tuple-functions.md | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 58613c6194b..853514c9743 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -606,6 +606,7 @@ tupleIntDiv(tuple_num, tuple_div) **Implementation details** - If either `tuple_num` or `tuple_div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor. +- An error will be thrown for division by 0. **Examples** @@ -659,13 +660,14 @@ tupleIntDivByNumber(tuple_num, div) **Implementation details** - If either `tuple_num` or `div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor. +- An error will be thrown for division by 0. **Examples** Query: ``` sql -SELECT tupleIntDivByNumber((15, 10, 5),5); +SELECT tupleIntDivByNumber((15, 10, 5), 5); ``` Result: @@ -679,7 +681,7 @@ Result: Query: ``` sql -SELECT tupleIntDivByNumber((15.2, 10.7, 5.5),5.8); +SELECT tupleIntDivByNumber((15.2, 10.7, 5.5), 5.8); ``` Result: @@ -690,6 +692,44 @@ Result: └─────────────────────────────────────────────┘ ``` +## tupleIntDivOrZero + +Like [tupleIntDiv](#tupleintdiv) it does integer division of a tuple of numerators and a tuple of denominators, and returns a tuple of the quotients. Does not throw an error for 0 divisors, but rather returns the quotient as 0. + +**Syntax** + +```sql +tupleIntDivOrZero(tuple_num, tuple_div) +``` + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `tuple_div`: Tuple of divisor values. [Tuple](../data-types/tuple) of numeric type. + +**Returned value** + +- Tuple of the quotients of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of integer values. +- Returns 0 for quotients where the divisor is 0. + +**Implementation details** + +- If either `tuple_num` or `tuple_div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor as in [tupleIntDiv](#tupleintdiv). + +**Examples** + +Query: + +``` sql +SELECT tupleIntDivOrZero((5, 10, 15),(0, 0, 0)); +``` + +Result: + +``` text +┌─tupleIntDivOrZero((5, 10, 15), (0, 0, 0))─┐ +│ (0,0,0) │ +└───────────────────────────────────────────┘ +``` + ## Distance functions All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md). From bba66eb05a95541915fd6e4eccd0beac26b16a3b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 9 Apr 2024 12:30:07 +0000 Subject: [PATCH 1126/1165] Review fixes. --- src/Analyzer/Utils.cpp | 22 ++++++++++++++++ src/Analyzer/Utils.h | 3 +++ src/Planner/CollectSets.cpp | 34 +++---------------------- src/Planner/CollectSets.h | 4 --- src/Storages/buildQueryTreeForShard.cpp | 5 ++-- 5 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index 8ccf95deadc..2882c4e0c02 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -760,4 +760,26 @@ QueryTreeNodePtr createCastFunction(QueryTreeNodePtr node, DataTypePtr result_ty return function_node; } +QueryTreeNodePtr buildSubqueryToReadColumnsFromTableExpression(QueryTreeNodePtr table_node, const ContextPtr & context) +{ + const auto & storage_snapshot = table_node->as()->getStorageSnapshot(); + auto columns_to_select = storage_snapshot->getColumns(GetColumnsOptions(GetColumnsOptions::Ordinary)); + size_t columns_to_select_size = columns_to_select.size(); + auto column_nodes_to_select = std::make_shared(); + column_nodes_to_select->getNodes().reserve(columns_to_select_size); + NamesAndTypes projection_columns; + projection_columns.reserve(columns_to_select_size); + for (auto & column : columns_to_select) + { + column_nodes_to_select->getNodes().emplace_back(std::make_shared(column, table_node)); + projection_columns.emplace_back(column.name, column.type); + } + auto subquery_for_table = std::make_shared(Context::createCopy(context)); + subquery_for_table->setIsSubquery(true); + subquery_for_table->getProjectionNode() = std::move(column_nodes_to_select); + subquery_for_table->getJoinTree() = std::move(table_node); + subquery_for_table->resolveProjectionColumns(std::move(projection_columns)); + return subquery_for_table; +} + } diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 8e32ef0464c..1b4a7d5ef3c 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -105,4 +105,7 @@ NameSet collectIdentifiersFullNames(const QueryTreeNodePtr & node); /// Wrap node into `_CAST` function QueryTreeNodePtr createCastFunction(QueryTreeNodePtr node, DataTypePtr result_type, ContextPtr context); +/// Build subquery which we execute for `IN table` function. +QueryTreeNodePtr buildSubqueryToReadColumnsFromTableExpression(QueryTreeNodePtr table_node, const ContextPtr & context); + } diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index 37502828f63..b1f2875210d 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -23,35 +23,6 @@ namespace ErrorCodes { extern const int UNSUPPORTED_METHOD; } - -QueryTreeNodePtr makeExecutableSubqueryForIn(const QueryTreeNodePtr & in_second_argument, const ContextPtr & context) -{ - auto subquery_to_execute = in_second_argument; - if (auto * table_node = in_second_argument->as()) - { - auto storage_snapshot = table_node->getStorageSnapshot(); - auto columns_to_select = storage_snapshot->getColumns(GetColumnsOptions(GetColumnsOptions::Ordinary)); - size_t columns_to_select_size = columns_to_select.size(); - auto column_nodes_to_select = std::make_shared(); - column_nodes_to_select->getNodes().reserve(columns_to_select_size); - NamesAndTypes projection_columns; - projection_columns.reserve(columns_to_select_size); - for (auto & column : columns_to_select) - { - column_nodes_to_select->getNodes().emplace_back(std::make_shared(column, subquery_to_execute)); - projection_columns.emplace_back(column.name, column.type); - } - auto subquery_for_table = std::make_shared(Context::createCopy(context)); - subquery_for_table->setIsSubquery(true); - subquery_for_table->getProjectionNode() = std::move(column_nodes_to_select); - subquery_for_table->getJoinTree() = std::move(subquery_to_execute); - subquery_for_table->resolveProjectionColumns(std::move(projection_columns)); - subquery_to_execute = std::move(subquery_for_table); - } - - return subquery_to_execute; -} - namespace { @@ -116,7 +87,10 @@ public: if (sets.findSubquery(set_key)) return; - auto subquery_to_execute = makeExecutableSubqueryForIn(in_second_argument, planner_context.getQueryContext()); + auto subquery_to_execute = in_second_argument; + if (in_second_argument->as()) + subquery_to_execute = buildSubqueryToReadColumnsFromTableExpression(std::move(subquery_to_execute), planner_context.getQueryContext()); + sets.addFromSubquery(set_key, std::move(subquery_to_execute), settings); } else diff --git a/src/Planner/CollectSets.h b/src/Planner/CollectSets.h index 0ee006f3320..5f9f7a5a466 100644 --- a/src/Planner/CollectSets.h +++ b/src/Planner/CollectSets.h @@ -14,8 +14,4 @@ struct SelectQueryOptions; */ void collectSets(const QueryTreeNodePtr & node, PlannerContext & planner_context); -/// Build subquery which we execute for IN function. -/// It is needed to support `IN table` case. -QueryTreeNodePtr makeExecutableSubqueryForIn(const QueryTreeNodePtr & in_second_argument, const ContextPtr & context); - } diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 09e48a93df4..5bbdbe487b0 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -365,7 +364,9 @@ QueryTreeNodePtr buildQueryTreeForShard(const PlannerContextPtr & planner_contex if (in_function_node_type != QueryTreeNodeType::QUERY && in_function_node_type != QueryTreeNodeType::UNION && in_function_node_type != QueryTreeNodeType::TABLE) continue; - auto subquery_to_execute = makeExecutableSubqueryForIn(in_function_subquery_node, planner_context->getQueryContext()); + auto subquery_to_execute = in_function_subquery_node; + if (subquery_to_execute->as()) + subquery_to_execute = buildSubqueryToReadColumnsFromTableExpression(std::move(subquery_to_execute), planner_context->getQueryContext()); auto temporary_table_expression_node = executeSubqueryNode(subquery_to_execute, planner_context->getMutableQueryContext(), From 92a027a2d0b5c92c7308925e314728588d8e52fa Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 15:01:27 +0200 Subject: [PATCH 1127/1165] Add OrZero variants --- .../functions/tuple-functions.md | 70 ++++++++++++++++--- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 853514c9743..2351cfd47d4 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -638,6 +638,44 @@ Result: └───────────────────────────────────────────┘ ``` +## tupleIntDivOrZero + +Like [tupleIntDiv](#tupleintdiv) it does integer division of a tuple of numerators and a tuple of denominators, and returns a tuple of the quotients. It does not throw an error for 0 divisors, but rather returns the quotient as 0. + +**Syntax** + +```sql +tupleIntDivOrZero(tuple_num, tuple_div) +``` + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `tuple_div`: Tuple of divisor values. [Tuple](../data-types/tuple) of numeric type. + +**Returned value** + +- Tuple of the quotients of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of integer values. +- Returns 0 for quotients where the divisor is 0. + +**Implementation details** + +- If either `tuple_num` or `tuple_div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor as in [tupleIntDiv](#tupleintdiv). + +**Examples** + +Query: + +``` sql +SELECT tupleIntDivOrZero((5, 10, 15),(0, 0, 0)); +``` + +Result: + +``` text +┌─tupleIntDivOrZero((5, 10, 15), (0, 0, 0))─┐ +│ (0,0,0) │ +└───────────────────────────────────────────┘ +``` + ## tupleIntDivByNumber Does integer division of a tuple of numerators by a given denominator, and returns a tuple of the quotients. @@ -692,40 +730,56 @@ Result: └─────────────────────────────────────────────┘ ``` -## tupleIntDivOrZero +## tupleIntDivOrZeroByNumber -Like [tupleIntDiv](#tupleintdiv) it does integer division of a tuple of numerators and a tuple of denominators, and returns a tuple of the quotients. Does not throw an error for 0 divisors, but rather returns the quotient as 0. +Like [tupleIntDivByNumber](#tupleintdivbynumber) it does integer division of a tuple of numerators by a given denominator, and returns a tuple of the quotients. It does not throw an error for 0 divisors, but rather returns the quotient as 0. **Syntax** ```sql -tupleIntDivOrZero(tuple_num, tuple_div) +tupleIntDivOrZeroByNumber(tuple_num, div) ``` +**Parameters** + - `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. -- `tuple_div`: Tuple of divisor values. [Tuple](../data-types/tuple) of numeric type. +- `div`: The divisor value. [Tuple](../data-types/tuple) of numeric type. **Returned value** -- Tuple of the quotients of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of integer values. +- Tuple of the quotients of `tuple_num` and `div`. [Tuple](../data-types/tuple) of integer values. - Returns 0 for quotients where the divisor is 0. **Implementation details** -- If either `tuple_num` or `tuple_div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor as in [tupleIntDiv](#tupleintdiv). +- If either `tuple_num` or `div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor as in [tupleIntDivByNumber](#tupleintdivbynumber). **Examples** Query: ``` sql -SELECT tupleIntDivOrZero((5, 10, 15),(0, 0, 0)); +SELECT tupleIntDivOrZeroByNumber((15, 10, 5),(5)); ``` Result: ``` text -┌─tupleIntDivOrZero((5, 10, 15), (0, 0, 0))─┐ +┌─tupleIntDivOrZeroByNumber((15, 10, 5), 5)─┐ +│ (3,2,1) │ +└───────────────────────────────────────────┘ +``` + +Query: + +``` sql +SELECT tupleIntDivOrZeroByNumber((15, 10, 5),(0)) +``` + +Result: + +``` text +┌─tupleIntDivOrZeroByNumber((15, 10, 5), 0)─┐ │ (0,0,0) │ └───────────────────────────────────────────┘ ``` From 3900b95fafee1b812522f45e25d13b259488e14c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 9 Apr 2024 13:01:38 +0000 Subject: [PATCH 1128/1165] Another one case with aliases. --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 21 ++++++++----------- ...23_group_by_use_nulls_analyzer_crashes.sql | 2 ++ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 793cec91293..487b40a5ccf 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -6694,11 +6694,8 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR { for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes()) { - for (auto & group_by_elem : grouping_set->as()->getNodes()) - { - group_by_elem = group_by_elem->clone(); + for (const auto & group_by_elem : grouping_set->as()->getNodes()) scope.nullable_group_by_keys.insert(group_by_elem); - } } } } @@ -6716,15 +6713,8 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR if (scope.group_by_use_nulls) { - for (auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) - { - /// Clone is needed cause aliases share subtrees. - /// If not clone, a part of GROUP BY key could be replaced to nullable - /// by replacing a part of alias from another subtree to nullable. - /// See 03023_group_by_use_nulls_analyzer_crashes - group_by_elem = group_by_elem->clone(); + for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) scope.nullable_group_by_keys.insert(group_by_elem); - } } } } @@ -8037,7 +8027,14 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier resolveGroupByNode(query_node_typed, scope); if (scope.group_by_use_nulls) + { resolved_expressions.clear(); + /// Clone is needed cause aliases share subtrees. + /// If not clone, the same (shared) subtree could be resolved again with different (Nullable) type + /// See 03023_group_by_use_nulls_analyzer_crashes + for (auto & [_, node] : scope.alias_name_to_expression_node) + node = node->clone(); + } if (query_node_typed.hasHaving()) resolveExpressionNode(query_node_typed.getHaving(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql index b6ca454c1cf..53882f115ba 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql @@ -17,3 +17,5 @@ GROUP BY (number, (toString(x), number)) SETTINGS group_by_use_nulls = 1 FORMAT Null; SELECT tuple(number + 1) AS x FROM numbers(10) GROUP BY number + 1, toString(x) WITH CUBE settings group_by_use_nulls=1 FORMAT Null; + +SELECT tuple(tuple(number)) AS x FROM numbers(10) WHERE toString(toUUID(tuple(number), NULL), x) GROUP BY number, (toString(x), number) WITH CUBE SETTINGS group_by_use_nulls = 1 FORMAT Null; From 7d969dd9faad1226eb60876bff5aa0dcfad5bc93 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 15:19:31 +0200 Subject: [PATCH 1129/1165] Add tupleModulo tupleModuloByNumber --- .../functions/tuple-functions.md | 86 +++++++++++++++++-- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 2351cfd47d4..ba177ca3349 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -613,7 +613,7 @@ tupleIntDiv(tuple_num, tuple_div) Query: ``` sql -SELECT tupleIntDiv((15, 10, 5),(5, 5, 5)); +SELECT tupleIntDiv((15, 10, 5), (5, 5, 5)); ``` Result: @@ -627,7 +627,7 @@ Result: Query: ``` sql -SELECT tupleIntDiv((15, 10, 5),(5.5, 5.5, 5.5)); +SELECT tupleIntDiv((15, 10, 5), (5.5, 5.5, 5.5)); ``` Result: @@ -665,7 +665,7 @@ tupleIntDivOrZero(tuple_num, tuple_div) Query: ``` sql -SELECT tupleIntDivOrZero((5, 10, 15),(0, 0, 0)); +SELECT tupleIntDivOrZero((5, 10, 15), (0, 0, 0)); ``` Result: @@ -689,7 +689,7 @@ tupleIntDivByNumber(tuple_num, div) **Parameters** - `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. -- `div`: The divisor value. [Tuple](../data-types/tuple) of numeric type. +- `div`: The divisor value. [Numeric](../data-types/int-uint.md) type. **Returned value** @@ -743,7 +743,7 @@ tupleIntDivOrZeroByNumber(tuple_num, div) **Parameters** - `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. -- `div`: The divisor value. [Tuple](../data-types/tuple) of numeric type. +- `div`: The divisor value. [Numeric](../data-types/int-uint.md) type. **Returned value** @@ -759,7 +759,7 @@ tupleIntDivOrZeroByNumber(tuple_num, div) Query: ``` sql -SELECT tupleIntDivOrZeroByNumber((15, 10, 5),(5)); +SELECT tupleIntDivOrZeroByNumber((15, 10, 5), (5)); ``` Result: @@ -773,7 +773,7 @@ Result: Query: ``` sql -SELECT tupleIntDivOrZeroByNumber((15, 10, 5),(0)) +SELECT tupleIntDivOrZeroByNumber((15, 10, 5), (0)) ``` Result: @@ -784,6 +784,78 @@ Result: └───────────────────────────────────────────┘ ``` +## tupleModulo + +Returns a tuple of the moduli (remainders) of division operations of two tuples. + +**Syntax** + +```sql +tupleModulo(tuple_num, tuple_mod) +``` + +**Parameters** + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `tuple_div`: Tuple of modulus values. [Tuple](../data-types/tuple) of numeric type. + +**Returned value** + +- Tuple of the remainders of division of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of non-zero integer values. +- An error is thrown for division by zero. + +**Examples** + +Query: + +``` sql +SELECT tupleModulo((15, 10, 5), (5, 3, 2)); +``` + +Result: + +``` text +┌─tupleModulo((15, 10, 5), (5, 3, 2))─┐ +│ (0,1,1) │ +└─────────────────────────────────────┘ +``` + +## tupleModuloByNumber + +Returns a tuple of the moduli (remainders) of division operations of a tuple and a given divisor. + +**Syntax** + +```sql +tupleModuloByNumber(tuple_num, div) +``` + +**Parameters** + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `div`: The divisor value. [Numeric](../data-types/int-uint.md) type. + +**Returned value** + +- Tuple of the remainders of division of `tuple_num` and `div`. [Tuple](../data-types/tuple) of non-zero integer values. +- An error is thrown for division by zero. + +**Examples** + +Query: + +``` sql +SELECT tupleModuloByNumber((15, 10, 5), 2); +``` + +Result: + +``` text +┌─tupleModuloByNumber((15, 10, 5), 2)─┐ +│ (1,0,1) │ +└─────────────────────────────────────┘ +``` + ## Distance functions All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md). From 5caa89a2b33efa86fab1bc6a6813e143c8f37f67 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Apr 2024 15:36:34 +0200 Subject: [PATCH 1130/1165] Fix completion of available ClickHouse tools Now clickhouse --help/ch --help will print --help for clickhouse-local, let's use just "clickhouse help" to get help with list of available tools in clickhouse binary itself. Signed-off-by: Azat Khuzhin --- programs/bash-completion/completions/clickhouse | 2 +- programs/main.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/bash-completion/completions/clickhouse b/programs/bash-completion/completions/clickhouse index ff0a60c60be..3c895a66075 100644 --- a/programs/bash-completion/completions/clickhouse +++ b/programs/bash-completion/completions/clickhouse @@ -3,7 +3,7 @@ function _clickhouse_get_utils() { local cmd=$1 && shift - "$cmd" --help |& awk '/^clickhouse.*args/ { print $2 }' + "$cmd" help |& awk '/^clickhouse.*args/ { print $2 }' } function _complete_for_clickhouse_entrypoint_bin() diff --git a/programs/main.cpp b/programs/main.cpp index 7162a18d764..9ad8b016c82 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -487,7 +487,7 @@ int main(int argc_, char ** argv_) /// Interpret binary without argument or with arguments starts with dash /// ('-') as clickhouse-local for better usability: /// - /// clickhouse # dumps help + /// clickhouse help # dumps help /// clickhouse -q 'select 1' # use local /// clickhouse # spawn local /// clickhouse local # spawn local From e07a614006cd756e3e86b01d22e954ca83ef4143 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 15:52:59 +0200 Subject: [PATCH 1131/1165] Add missing tests for tupleIntXYZ and tupleModulo, tupleModuloByNumber --- .../03033_tupleIntXYZ_and_tupleModulo.reference | 9 +++++++++ .../03033_tupleIntXYZ_and_tupleModulo.sql | 13 +++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/03033_tupleIntXYZ_and_tupleModulo.reference create mode 100644 tests/queries/0_stateless/03033_tupleIntXYZ_and_tupleModulo.sql diff --git a/tests/queries/0_stateless/03033_tupleIntXYZ_and_tupleModulo.reference b/tests/queries/0_stateless/03033_tupleIntXYZ_and_tupleModulo.reference new file mode 100644 index 00000000000..f0ce1649939 --- /dev/null +++ b/tests/queries/0_stateless/03033_tupleIntXYZ_and_tupleModulo.reference @@ -0,0 +1,9 @@ +(3,2,1) +(2,1,0) +(0,0,0) +(3,2,1) +(2,1,0) +(3,2,1) +(0,0,0) +(0,1,1) +(1,0,1) diff --git a/tests/queries/0_stateless/03033_tupleIntXYZ_and_tupleModulo.sql b/tests/queries/0_stateless/03033_tupleIntXYZ_and_tupleModulo.sql new file mode 100644 index 00000000000..2cb7e726a4b --- /dev/null +++ b/tests/queries/0_stateless/03033_tupleIntXYZ_and_tupleModulo.sql @@ -0,0 +1,13 @@ +SELECT tupleIntDiv((15, 10, 5), (0, 0, 0)); -- { serverError ILLEGAL_DIVISION } +SELECT tupleIntDiv((15, 10, 5), (5, 5, 5)); +SELECT tupleIntDiv((15, 10, 5), (5.5, 5.5, 5.5)); +SELECT tupleIntDivOrZero((5, 10, 15), (0, 0, 0)); -- no error thrown for zero divisors +SELECT tupleIntDivByNumber((15, 10, 5), 0); -- { serverError ILLEGAL_DIVISION } +SELECT tupleIntDivByNumber((15, 10, 5), 5); +SELECT tupleIntDivByNumber((15.2, 10.7, 5.5), 5.8); +SELECT tupleIntDivOrZeroByNumber((15, 10, 5), 5); +SELECT tupleIntDivOrZeroByNumber((15, 10, 5), 0); -- no error thrown for zero divisors +SELECT tupleModulo((15, 10, 5), (0, 3, 2)); -- { serverError ILLEGAL_DIVISION } +SELECT tupleModulo((15, 10, 5), (5, 3, 2)); +SELECT tupleModuloByNumber((15, 10, 5), 0); -- { serverError ILLEGAL_DIVISION } +SELECT tupleModuloByNumber((15, 10, 5), 2); \ No newline at end of file From 42a906dca9be03d3380fe165bc3e0fddad90c7e0 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 8 Apr 2024 21:41:06 +0800 Subject: [PATCH 1132/1165] Remove useless param, fix typo and query result --- docs/en/getting-started/example-datasets/opensky.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/getting-started/example-datasets/opensky.md b/docs/en/getting-started/example-datasets/opensky.md index df28809495c..9f8ad134899 100644 --- a/docs/en/getting-started/example-datasets/opensky.md +++ b/docs/en/getting-started/example-datasets/opensky.md @@ -7,7 +7,7 @@ title: "Crowdsourced air traffic data from The OpenSky Network 2020" The data in this dataset is derived and cleaned from the full OpenSky dataset to illustrate the development of air traffic during the COVID-19 pandemic. It spans all flights seen by the network's more than 2500 members since 1 January 2019. More data will be periodically included in the dataset until the end of the COVID-19 pandemic. -Source: https://zenodo.org/record/5092942#.YRBCyTpRXYd +Source: https://zenodo.org/records/5092942 Martin Strohmeier, Xavier Olive, Jannis Luebbe, Matthias Schaefer, and Vincent Lenders "Crowdsourced air traffic data from the OpenSky Network 2019–2020" @@ -19,7 +19,7 @@ https://doi.org/10.5194/essd-13-357-2021 Run the command: ```bash -wget -O- https://zenodo.org/record/5092942 | grep -oP 'https://zenodo.org/record/5092942/files/flightlist_\d+_\d+\.csv\.gz' | xargs wget +wget -O- https://zenodo.org/records/5092942 | grep -oE 'https://zenodo.org/records/5092942/files/flightlist_[0-9]+_[0-9]+\.csv\.gz' | xargs wget ``` Download will take about 2 minutes with good internet connection. There are 30 files with total size of 4.3 GB. @@ -134,7 +134,7 @@ Result: ```text ┌─avg(geoDistance(longitude_1, latitude_1, longitude_2, latitude_2))─┐ -│ 1041090.6465708319 │ +│ 1041090.6360469435 │ └────────────────────────────────────────────────────────────────────┘ ``` From de8d31685db079da0c4e734330a3d75a67a30c5a Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 16:26:35 +0200 Subject: [PATCH 1133/1165] Minor edit --- docs/en/sql-reference/functions/tuple-functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index ba177ca3349..b3cec1206b8 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -759,7 +759,7 @@ tupleIntDivOrZeroByNumber(tuple_num, div) Query: ``` sql -SELECT tupleIntDivOrZeroByNumber((15, 10, 5), (5)); +SELECT tupleIntDivOrZeroByNumber((15, 10, 5), 5); ``` Result: @@ -773,7 +773,7 @@ Result: Query: ``` sql -SELECT tupleIntDivOrZeroByNumber((15, 10, 5), (0)) +SELECT tupleIntDivOrZeroByNumber((15, 10, 5), 0) ``` Result: From d5014b2d0e3f19aba82cca1480146cec1772e7a0 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 16:44:07 +0200 Subject: [PATCH 1134/1165] Add missing L2SquaredNorm function --- .../functions/distance-functions.md | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/en/sql-reference/functions/distance-functions.md b/docs/en/sql-reference/functions/distance-functions.md index e20c35c6b6f..eb991acc94b 100644 --- a/docs/en/sql-reference/functions/distance-functions.md +++ b/docs/en/sql-reference/functions/distance-functions.md @@ -82,6 +82,44 @@ Result: └──────────────────┘ ``` +## L2SquaredNorm + +Calculates the square root of the sum of the squares of the vector values (the [L2Norm](#l2norm)) squared. + +**Syntax** + +```sql +L2SquaredNorm(vector) +``` + +Alias: `normL2Squared`. + +***Arguments** + +- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- L2-norm squared. + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT L2SquaredNorm((1, 2)); +``` + +Result: + +```text +┌─L2SquaredNorm((1, 2))─┐ +│ 5 │ +└───────────────────────┘ +``` + ## LinfNorm Calculates the maximum of absolute values of a vector. From 728ed2eee3ee08520fa0e36c81ad5e8c0c5472fc Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 9 Apr 2024 16:44:38 +0200 Subject: [PATCH 1135/1165] Simplify the change --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 96 +++++++++++------------ 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 48f32652198..5aaf5bc52b6 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2276,61 +2276,61 @@ void QueryAnalyzer::mergeWindowWithParentWindow(const QueryTreeNodePtr & window_ void QueryAnalyzer::replaceNodesWithPositionalArguments(QueryTreeNodePtr & node_list, const QueryTreeNodes & projection_nodes, IdentifierResolveScope & scope) { const auto & settings = scope.context->getSettingsRef(); - if (settings.enable_positional_arguments && scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + if (!settings.enable_positional_arguments || !scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + return; + + auto & node_list_typed = node_list->as(); + + for (auto & node : node_list_typed.getNodes()) { - auto & node_list_typed = node_list->as(); + auto * node_to_replace = &node; - for (auto & node : node_list_typed.getNodes()) + if (auto * sort_node = node->as()) + node_to_replace = &sort_node->getExpression(); + + auto * constant_node = (*node_to_replace)->as(); + + if (!constant_node + || (constant_node->getValue().getType() != Field::Types::UInt64 + && constant_node->getValue().getType() != Field::Types::Int64)) + continue; + + UInt64 pos; + if (constant_node->getValue().getType() == Field::Types::UInt64) { - auto * node_to_replace = &node; - - if (auto * sort_node = node->as()) - node_to_replace = &sort_node->getExpression(); - - auto * constant_node = (*node_to_replace)->as(); - - if (!constant_node - || (constant_node->getValue().getType() != Field::Types::UInt64 - && constant_node->getValue().getType() != Field::Types::Int64)) - continue; - - UInt64 pos; - if (constant_node->getValue().getType() == Field::Types::UInt64) + pos = constant_node->getValue().get(); + } + else // Int64 + { + auto value = constant_node->getValue().get(); + if (value > 0) + pos = value; + else { - pos = constant_node->getValue().get(); - } - else // Int64 - { - auto value = constant_node->getValue().get(); - if (value > 0) - pos = value; - else - { - if (static_cast(std::abs(value)) > projection_nodes.size()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Negative positional argument number {} is out of bounds. Expected in range [-{}, -1]. In scope {}", - value, - projection_nodes.size(), - scope.scope_node->formatASTForErrorMessage()); - pos = projection_nodes.size() + value + 1; - } + if (static_cast(std::abs(value)) > projection_nodes.size()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Negative positional argument number {} is out of bounds. Expected in range [-{}, -1]. In scope {}", + value, + projection_nodes.size(), + scope.scope_node->formatASTForErrorMessage()); + pos = projection_nodes.size() + value + 1; } + } - if (!pos || pos > projection_nodes.size()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Positional argument number {} is out of bounds. Expected in range [1, {}]. In scope {}", - pos, - projection_nodes.size(), - scope.scope_node->formatASTForErrorMessage()); + if (!pos || pos > projection_nodes.size()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Positional argument number {} is out of bounds. Expected in range [1, {}]. In scope {}", + pos, + projection_nodes.size(), + scope.scope_node->formatASTForErrorMessage()); - --pos; - *node_to_replace = projection_nodes[pos]->clone(); - if (auto it = resolved_expressions.find(projection_nodes[pos]); it != resolved_expressions.end()) - { - resolved_expressions[*node_to_replace] = it->second; - } + --pos; + *node_to_replace = projection_nodes[pos]->clone(); + if (auto it = resolved_expressions.find(projection_nodes[pos]); it != resolved_expressions.end()) + { + resolved_expressions[*node_to_replace] = it->second; } } } From 8369f8d8c18aea16a815a229f1bfd4af27a9f102 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 16:54:09 +0200 Subject: [PATCH 1136/1165] Add missing l2SquaredNorm function --- .../functions/distance-functions.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/en/sql-reference/functions/distance-functions.md b/docs/en/sql-reference/functions/distance-functions.md index e20c35c6b6f..5f3514049c7 100644 --- a/docs/en/sql-reference/functions/distance-functions.md +++ b/docs/en/sql-reference/functions/distance-functions.md @@ -81,6 +81,43 @@ Result: │ 2.23606797749979 │ └──────────────────┘ ``` +## L2SquaredNorm + +Calculates the square root of the sum of the squares of the vector values (the [L2Norm](#l2norm)) squared. + +**Syntax** + +```sql +L2SquaredNorm(vector) +``` + +Alias: `normL2Squared`. + +***Arguments** + +- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- L2-norm squared. + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT L2SquaredNorm((1, 2)); +``` + +Result: + +```text +┌─L2SquaredNorm((1, 2))─┐ +│ 5 │ +└───────────────────────┘ +``` ## LinfNorm From 814de46e136f14a1760f1045f4d08a44e082a42c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 9 Apr 2024 15:08:03 +0000 Subject: [PATCH 1137/1165] Another one case. --- src/Analyzer/HashUtils.h | 2 +- tests/queries/0_stateless/02203_shebang.bak | 3 + ...up_by_use_nulls_analyzer_crashes.reference | 4 + ...23_group_by_use_nulls_analyzer_crashes.sql | 2 + tests/queries/0_stateless/users.xml | 110 ++++++++++++++++++ 5 files changed, 120 insertions(+), 1 deletion(-) create mode 100755 tests/queries/0_stateless/02203_shebang.bak create mode 100644 tests/queries/0_stateless/users.xml diff --git a/src/Analyzer/HashUtils.h b/src/Analyzer/HashUtils.h index 80f59c1eaaa..77ade7a4705 100644 --- a/src/Analyzer/HashUtils.h +++ b/src/Analyzer/HashUtils.h @@ -36,7 +36,7 @@ inline bool operator!=(const QueryTreeNodeWithHash; -using QueryTreeNodePtrWithHashIgnoreTypes = QueryTreeNodeWithHash; +using QueryTreeNodePtrWithHashIgnoreTypes = QueryTreeNodeWithHash; using QueryTreeNodeRawPtrWithHash = QueryTreeNodeWithHash; using QueryTreeNodeConstRawPtrWithHash = QueryTreeNodeWithHash; diff --git a/tests/queries/0_stateless/02203_shebang.bak b/tests/queries/0_stateless/02203_shebang.bak new file mode 100755 index 00000000000..07686d1aab4 --- /dev/null +++ b/tests/queries/0_stateless/02203_shebang.bak @@ -0,0 +1,3 @@ +#!/usr/bin/clickhouse-local --queries-file + +SELECT 1; diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference index 4243abb1a1e..17a17484a0c 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.reference @@ -62,3 +62,7 @@ (9) a b a b +a a +a a + +a a diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql index 53882f115ba..68710137542 100644 --- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql +++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql @@ -19,3 +19,5 @@ SETTINGS group_by_use_nulls = 1 FORMAT Null; SELECT tuple(number + 1) AS x FROM numbers(10) GROUP BY number + 1, toString(x) WITH CUBE settings group_by_use_nulls=1 FORMAT Null; SELECT tuple(tuple(number)) AS x FROM numbers(10) WHERE toString(toUUID(tuple(number), NULL), x) GROUP BY number, (toString(x), number) WITH CUBE SETTINGS group_by_use_nulls = 1 FORMAT Null; + +SELECT materialize('a'), 'a' AS key GROUP BY key WITH CUBE WITH TOTALS SETTINGS group_by_use_nulls = 1; diff --git a/tests/queries/0_stateless/users.xml b/tests/queries/0_stateless/users.xml new file mode 100644 index 00000000000..a199435b42f --- /dev/null +++ b/tests/queries/0_stateless/users.xml @@ -0,0 +1,110 @@ + + + + + + + + + + + + 1 + + + + + + + + + c64c5e4e53ea1a9f1427d2713b3a22bbebe8940bc807adaf654744b1568c70ab + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + From 283fc115ba3c57a12fbacb8afd0af7cc332722f9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 9 Apr 2024 15:17:26 +0000 Subject: [PATCH 1138/1165] Updating the test. --- tests/integration/test_cluster_all_replicas/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_cluster_all_replicas/test.py b/tests/integration/test_cluster_all_replicas/test.py index 59b41ca87af..d8bad180e1b 100644 --- a/tests/integration/test_cluster_all_replicas/test.py +++ b/tests/integration/test_cluster_all_replicas/test.py @@ -43,7 +43,8 @@ def test_cluster(start_cluster): def test_global_in(start_cluster): - node1.query("CREATE TABLE u(uid Int16) ENGINE=Log as select 0") + node1.query("DROP TABLE IF EXISTS u;") + node1.query("CREATE TABLE u(uid Int16) ENGINE=Memory as select 0") assert set( node1.query( From 6feb2744672f417ffd5d4e0fff394a40af73ff61 Mon Sep 17 00:00:00 2001 From: flynn Date: Tue, 9 Apr 2024 15:18:57 +0000 Subject: [PATCH 1139/1165] Fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 5aaf5bc52b6..56ccd5c6c22 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2276,7 +2276,7 @@ void QueryAnalyzer::mergeWindowWithParentWindow(const QueryTreeNodePtr & window_ void QueryAnalyzer::replaceNodesWithPositionalArguments(QueryTreeNodePtr & node_list, const QueryTreeNodes & projection_nodes, IdentifierResolveScope & scope) { const auto & settings = scope.context->getSettingsRef(); - if (!settings.enable_positional_arguments || !scope.context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + if (!settings.enable_positional_arguments || scope.context->getClientInfo().query_kind != ClientInfo::QueryKind::INITIAL_QUERY) return; auto & node_list_typed = node_list->as(); From 0bb54101b718720b907229f28758355ec52670a3 Mon Sep 17 00:00:00 2001 From: serxa Date: Tue, 9 Apr 2024 15:35:36 +0000 Subject: [PATCH 1140/1165] Use shared mutex for stacktrace cache access --- src/Common/StackTrace.cpp | 67 ++++++++++++++------------------------- 1 file changed, 23 insertions(+), 44 deletions(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 78ab43e8991..4200161f8e8 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -480,10 +481,8 @@ void StackTrace::toStringEveryLine(void ** frame_pointers_raw, size_t offset, si struct CacheEntry { + std::mutex mutex; std::optional stacktrace_string; - bool to_string_in_progress = false; - - std::condition_variable cv; }; using CacheEntryPtr = std::shared_ptr; @@ -496,67 +495,47 @@ static StackTraceCache & cacheInstance() return cache; } -static std::mutex stacktrace_cache_mutex; +static DB::SharedMutex stacktrace_cache_mutex; String toStringCached(const StackTrace::FramePointers & pointers, size_t offset, size_t size) { const StackTraceRefTriple key{pointers, offset, size}; /// Calculation of stack trace text is extremely slow. - /// We use simple cache because otherwise the server could be overloaded by trash queries. + /// We use cache because otherwise the server could be overloaded by trash queries. /// Note that this cache can grow unconditionally, but practically it should be small. - std::unique_lock lock{stacktrace_cache_mutex}; - CacheEntryPtr cache_entry; StackTraceCache & cache = cacheInstance(); - if (auto it = cache.find(key); it != cache.end()) + CacheEntryPtr cache_entry; + + // Optimistic try for cache hit to avoid any contention whatsoever, should be the main hot code route { - cache_entry = it->second; - } - else - { - auto [new_it, inserted] = cache.emplace(StackTraceTriple{pointers, offset, size}, std::make_shared()); - chassert(inserted); - cache_entry = new_it->second; + std::shared_lock read_lock{stacktrace_cache_mutex}; + if (auto it = cache.find(key); it != cache.end()) + cache_entry = it->second; } - if (!cache_entry->to_string_in_progress && cache_entry->stacktrace_string.has_value()) - return *cache_entry->stacktrace_string; - - if (cache_entry->to_string_in_progress) + // Create a new entry in case of a cache miss + if (!cache_entry) { - cache_entry->cv.wait(lock, [&]{ return !cache_entry->to_string_in_progress; }); + std::unique_lock write_lock{stacktrace_cache_mutex}; - if (cache_entry->stacktrace_string.has_value()) - return *cache_entry->stacktrace_string; + // We should recheck because `shared_lock` was released before we acquired `write_lock` + if (auto it = cache.find(key); it != cache.end()) + cache_entry = it->second; // Another thread managed to created this entry before us + else + cache_entry = cache.emplace(StackTraceTriple{pointers, offset, size}, std::make_shared()).first->second; } - cache_entry->to_string_in_progress = true; - - lock.unlock(); - - String stacktrace_string; - try + // Do not hold `stacktrace_cache_mutex` while running possibly slow calculation of stack trace text + std::scoped_lock lock(cache_entry->mutex); + if (!cache_entry->stacktrace_string.has_value()) { DB::WriteBufferFromOwnString out; toStringEveryLineImpl(false, key, [&](std::string_view str) { out << str << '\n'; }); - stacktrace_string = out.str(); - } - catch (...) - { - lock.lock(); - cache_entry->to_string_in_progress = false; - lock.unlock(); - cache_entry->cv.notify_one(); - throw; + cache_entry->stacktrace_string = out.str(); } - lock.lock(); - cache_entry->to_string_in_progress = false; - cache_entry->stacktrace_string = stacktrace_string; - lock.unlock(); - - cache_entry->cv.notify_all(); - return stacktrace_string; + return *cache_entry->stacktrace_string; } std::string StackTrace::toString() const From c7cb33d035559db910f2a1fa6ea969941b09de7a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 9 Apr 2024 15:43:08 +0000 Subject: [PATCH 1141/1165] Better parsing --- src/Core/SettingsChangesHistory.h | 2 +- src/Interpreters/Cache/QueryCache.cpp | 23 +++++++++++-------- .../02494_query_cache_system_tables.sql | 13 +++++++++++ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 7fa12780c8c..8b5cdf03a33 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -87,7 +87,7 @@ static std::map sett { {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"}, - {"query_cache_system_table_handling", QueryCacheSystemTableHandling::Save, QueryCacheSystemTableHandling::Throw, "The query cache no longer caches results of queries against system tables"}, + {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"}, }}, {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index 67fcdb8159c..7b1f24e93fc 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -9,13 +9,15 @@ #include #include #include +#include +#include #include +#include #include #include #include #include #include -#include #include #include /// chassert @@ -86,16 +88,19 @@ struct HasSystemTablesMatcher /// Handle SELECT [...] FROM clusterAllReplicas(, '
') else if (const auto * literal = node->as()) { - const auto & value = literal->value; /// (*) - database_table = applyVisitor(FieldVisitorDump(), value); + const auto & value = literal->value; + database_table = toString(value); } - /// (*) returns table in quotes, so we can't use .starts_with() for matching - static const re2::RE2 is_system_table(String(DatabaseCatalog::TEMPORARY_DATABASE) - + "|" + DatabaseCatalog::SYSTEM_DATABASE - + "|" + DatabaseCatalog::INFORMATION_SCHEMA - + "|" + DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE); - data.has_system_tables = re2::RE2::PartialMatch(database_table, is_system_table); + Tokens tokens(database_table.c_str(), database_table.c_str() + database_table.size(), /*max_query_size*/ 2048, /*skip_insignificant*/ true); + IParser::Pos pos(tokens, /*max_depth*/ 42, /*max_backtracks*/ 42); + Expected expected; + String database; + String table; + bool successfully_parsed = parseDatabaseAndTableName(pos, expected, database, table); + if (successfully_parsed) + if (DatabaseCatalog::isPredefinedDatabase(database)) + data.has_system_tables = true; } }; diff --git a/tests/queries/0_stateless/02494_query_cache_system_tables.sql b/tests/queries/0_stateless/02494_query_cache_system_tables.sql index 935011a6bb0..c67a5c49dda 100644 --- a/tests/queries/0_stateless/02494_query_cache_system_tables.sql +++ b/tests/queries/0_stateless/02494_query_cache_system_tables.sql @@ -44,8 +44,21 @@ SELECT * SETTINGS use_query_cache = 1; SELECT * FROM information_schema.tables SETTINGS use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } SELECT * FROM INFORMATION_SCHEMA.TABLES SETTINGS use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } +-- System tables can be "hidden" inside e.g. table functions SELECT * FROM clusterAllReplicas('test_shard_localhost', system.one) SETTINGS use_query_cache = 1; -- {serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } SELECT * FROM clusterAllReplicas('test_shard_localhost', 'system.one') SETTINGS use_query_cache = 1; -- {serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } +-- Criminal edge case that a user creates a table named "system". The query cache must not reject queries against it. +DROP TABLE IF EXISTS system; +CREATE TABLE system (c UInt64) ENGINE = Memory; +SElECT * FROM system SETTINGS use_query_cache = 1; +DROP TABLE system; + +-- Similar queries against system.system are rejected. +DROP TABLE IF EXISTS system.system; +CREATE TABLE system.system (c UInt64) ENGINE = Memory; +SElECT * FROM system.system SETTINGS use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } +DROP TABLE system; + -- Cleanup SYSTEM DROP QUERY CACHE; From 7aac552fabfa9a2fe19ca6930000eeee395e8752 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 17:44:41 +0200 Subject: [PATCH 1142/1165] Add missing kostikConsistentHash --- .../sql-reference/functions/hash-functions.md | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 90c7d8c2206..5bfacd34e19 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -594,6 +594,45 @@ Calculates JumpConsistentHash form a UInt64. Accepts two arguments: a UInt64-type key and the number of buckets. Returns Int32. For more information, see the link: [JumpConsistentHash](https://arxiv.org/pdf/1406.2294.pdf) +## kostikConsistentHash + +An O(1) time and space consistent hash algorithm by Konstantin 'kostik' Oblakov. Previously `yandexConsistentHash`. + +**Syntax** + +```sql +kostikConsistentHash(input, n) +``` + +Alias: `yandexConsistentHash` (left for backwards compatibility sake). + +**Parameters** + +- `input`: A UInt64-type key [UInt64](/docs/en/sql-reference/data-types/int-uint.md). +- `n`: Number of buckets. [UInt16](/docs/en/sql-reference/data-types/int-uint.md). + +**Returned value** + +- A [UInt16](/docs/en/sql-reference/data-types/int-uint.md) data type hash value. + +**Implementation details** + +It is efficient only if n <= 32768. + +**Example** + +Query: + +```sql +SELECT kostikConsistentHash(16045690984833335023, 2); +``` + +```response +┌─kostikConsistentHash(16045690984833335023, 2)─┐ +│ 1 │ +└───────────────────────────────────────────────┘ +``` + ## murmurHash2_32, murmurHash2_64 Produces a [MurmurHash2](https://github.com/aappleby/smhasher) hash value. From 76ebaedfd29311770aef47a59acaf5212fef868a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 9 Apr 2024 15:45:35 +0000 Subject: [PATCH 1143/1165] Cosmetics --- tests/queries/0_stateless/02494_query_cache_system_tables.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02494_query_cache_system_tables.sql b/tests/queries/0_stateless/02494_query_cache_system_tables.sql index c67a5c49dda..7c9f01c4e91 100644 --- a/tests/queries/0_stateless/02494_query_cache_system_tables.sql +++ b/tests/queries/0_stateless/02494_query_cache_system_tables.sql @@ -54,11 +54,11 @@ CREATE TABLE system (c UInt64) ENGINE = Memory; SElECT * FROM system SETTINGS use_query_cache = 1; DROP TABLE system; --- Similar queries against system.system are rejected. +-- But queries against system.system are rejected. DROP TABLE IF EXISTS system.system; CREATE TABLE system.system (c UInt64) ENGINE = Memory; SElECT * FROM system.system SETTINGS use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_SYSTEM_TABLE } -DROP TABLE system; +DROP TABLE system.system; -- Cleanup SYSTEM DROP QUERY CACHE; From 9419c0f7882f8a99cd15115fdbc4946d9fe0e91c Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 17:53:00 +0200 Subject: [PATCH 1144/1165] remove l2squared --- .../functions/distance-functions.md | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/docs/en/sql-reference/functions/distance-functions.md b/docs/en/sql-reference/functions/distance-functions.md index eb991acc94b..e20c35c6b6f 100644 --- a/docs/en/sql-reference/functions/distance-functions.md +++ b/docs/en/sql-reference/functions/distance-functions.md @@ -82,44 +82,6 @@ Result: └──────────────────┘ ``` -## L2SquaredNorm - -Calculates the square root of the sum of the squares of the vector values (the [L2Norm](#l2norm)) squared. - -**Syntax** - -```sql -L2SquaredNorm(vector) -``` - -Alias: `normL2Squared`. - -***Arguments** - -- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). - -**Returned value** - -- L2-norm squared. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L2SquaredNorm((1, 2)); -``` - -Result: - -```text -┌─L2SquaredNorm((1, 2))─┐ -│ 5 │ -└───────────────────────┘ -``` - ## LinfNorm Calculates the maximum of absolute values of a vector. From 17d3d57f9f7ab4d915090c98b87c6e161f7ae81d Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:01:12 +0200 Subject: [PATCH 1145/1165] fix flaky result --- docs/en/getting-started/example-datasets/opensky.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/getting-started/example-datasets/opensky.md b/docs/en/getting-started/example-datasets/opensky.md index 9f8ad134899..c0b4d96725d 100644 --- a/docs/en/getting-started/example-datasets/opensky.md +++ b/docs/en/getting-started/example-datasets/opensky.md @@ -127,15 +127,15 @@ Average flight distance is around 1000 km. Query: ```sql -SELECT avg(geoDistance(longitude_1, latitude_1, longitude_2, latitude_2)) FROM opensky; +SELECT round(avg(geoDistance(longitude_1, latitude_1, longitude_2, latitude_2)), 2) FROM opensky; ``` Result: ```text -┌─avg(geoDistance(longitude_1, latitude_1, longitude_2, latitude_2))─┐ -│ 1041090.6360469435 │ -└────────────────────────────────────────────────────────────────────┘ + ┌─round(avg(geoDistance(longitude_1, latitude_1, longitude_2, latitude_2)), 2)─┐ +1. │ 1041090.67 │ -- 1.04 million + └──────────────────────────────────────────────────────────────────────────────┘ ``` ### Most busy origin airports and the average distance seen {#busy-airports-average-distance} From fcfaf82181f97d888e4ddccd0754c80e31e7f567 Mon Sep 17 00:00:00 2001 From: Jayme Bird Date: Tue, 9 Apr 2024 17:49:46 +0100 Subject: [PATCH 1146/1165] fix: add missing hostname column to blob_storage_log system table --- docs/en/operations/system-tables/blob_storage_log.md | 2 ++ src/Interpreters/BlobStorageLog.cpp | 3 +++ 2 files changed, 5 insertions(+) diff --git a/docs/en/operations/system-tables/blob_storage_log.md b/docs/en/operations/system-tables/blob_storage_log.md index 2328f7f0346..8c0c33a504a 100644 --- a/docs/en/operations/system-tables/blob_storage_log.md +++ b/docs/en/operations/system-tables/blob_storage_log.md @@ -7,6 +7,7 @@ Contains logging entries with information about various blob storage operations Columns: +- `hostname` ([LowCardinality(String)](../../sql-reference/data-types/string.md)) — Hostname of the server executing the query. - `event_date` ([Date](../../sql-reference/data-types/date.md)) — Date of the event. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time of the event. - `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Time of the event with microseconds precision. @@ -38,6 +39,7 @@ SELECT * FROM system.blob_storage_log WHERE query_id = '7afe0450-504d-4e4b-9a80- ```text Row 1: ────── +hostname: clickhouse.eu-central1.internal event_date: 2023-10-31 event_time: 2023-10-31 16:03:40 event_time_microseconds: 2023-10-31 16:03:40.481437 diff --git a/src/Interpreters/BlobStorageLog.cpp b/src/Interpreters/BlobStorageLog.cpp index f9d5b0d6790..0324ef8713c 100644 --- a/src/Interpreters/BlobStorageLog.cpp +++ b/src/Interpreters/BlobStorageLog.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -26,6 +27,7 @@ ColumnsDescription BlobStorageLogElement::getColumnsDescription() return ColumnsDescription { + {"hostname", std::make_shared(std::make_shared()), "Hostname of the server executing the query."}, {"event_date", std::make_shared(), "Date of the event."}, {"event_time", std::make_shared(), "Time of the event."}, {"event_time_microseconds", std::make_shared(6), "Time of the event with microseconds precision."}, @@ -51,6 +53,7 @@ void BlobStorageLogElement::appendToBlock(MutableColumns & columns) const size_t i = 0; auto event_time_seconds = timeInSeconds(event_time); + columns[i++]->insert(getFQDNOrHostName()); columns[i++]->insert(DateLUT::instance().toDayNum(event_time_seconds).toUnderType()); columns[i++]->insert(event_time_seconds); columns[i++]->insert(Decimal64(timeInMicroseconds(event_time))); From 98c1cc7747ba2a8afb18982666f550167a6557ee Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Tue, 9 Apr 2024 17:09:04 +0000 Subject: [PATCH 1147/1165] new gh runner version 2.315.0 --- tests/ci/worker/prepare-ci-ami.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/worker/prepare-ci-ami.sh b/tests/ci/worker/prepare-ci-ami.sh index 281dff5b1c2..effc224c2d5 100644 --- a/tests/ci/worker/prepare-ci-ami.sh +++ b/tests/ci/worker/prepare-ci-ami.sh @@ -9,7 +9,7 @@ set -xeuo pipefail echo "Running prepare script" export DEBIAN_FRONTEND=noninteractive -export RUNNER_VERSION=2.313.0 +export RUNNER_VERSION=2.315.0 export RUNNER_HOME=/home/ubuntu/actions-runner deb_arch() { From 077c57a4c9e56c387afffa1da2aaabe970fde305 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 20:14:50 +0200 Subject: [PATCH 1148/1165] Add functions and word moduli to aspell-dict --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 8aa2a463c47..146b9d48607 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1935,6 +1935,7 @@ mmap mmapped modularization moduloOrZero +moduli mongodb monthName moscow @@ -2646,6 +2647,12 @@ tupleMultiplyByNumber tupleNegate tuplePlus tupleToNameValuePairs +tupleIntDiv +tupleIntDivByNumber +tupleIntDivOrZero +tupleIntDivOrZeroByNumber +tupleModulo +tupleModuloByNumber turbostat txt typename From 9a6b987b0088b6173412da57a12ec4f3d86e0234 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 20:16:59 +0200 Subject: [PATCH 1149/1165] Add kostikConstantHash related words to aspell-dict --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 8aa2a463c47..005a5a7c69b 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -452,6 +452,9 @@ Khanna KittenHouse Klickhouse Kolmogorov +Konstantin +kostik +kostikConsistentHash Korzeniewski Kubernetes LDAP @@ -655,6 +658,7 @@ OTLP OUTFILE ObjectId Observability +Oblakov Octonica Ok OnTime From 9b35c637c173bc64f15ae6978e66966bc2b17ab8 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Apr 2024 20:19:47 +0200 Subject: [PATCH 1150/1165] Add SquaredNorm to aspell-ignore --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 8aa2a463c47..1da1373070c 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -860,6 +860,7 @@ Soundex SpanKind Spearman's SquaredDistance +SquaredNorm StartTLS StartTime StartupSystemTables From de2a5f018f4891619fd74eb929b998652b615f83 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 9 Apr 2024 20:40:40 +0200 Subject: [PATCH 1151/1165] Update CollectSets.cpp --- src/Planner/CollectSets.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index b1f2875210d..f00b1e6ab16 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -23,6 +23,7 @@ namespace ErrorCodes { extern const int UNSUPPORTED_METHOD; } + namespace { From 0e7d05e0912ddbab866589b65ab5966589860f62 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 10 Apr 2024 01:58:30 +0200 Subject: [PATCH 1152/1165] Fix UBSan --- src/Client/ClientBaseHelpers.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 3f3e3b1492f..b1d29b34ffc 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -6,6 +6,8 @@ #include #include +#include + namespace DB { @@ -173,7 +175,8 @@ void highlight(const String & query, std::vector & colors /// Highlight the last error in red. If the parser failed or the lexer found an invalid token, /// or if it didn't parse all the data (except, the data for INSERT query, which is legitimately unparsed) if ((!parse_res || last_token.isError() || (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon)) - && !(insert_data && expected.max_parsed_pos >= insert_data)) + && !(insert_data && expected.max_parsed_pos >= insert_data) + && expected.max_parsed_pos >= prev) { pos += UTF8::countCodePoints(reinterpret_cast(prev), expected.max_parsed_pos - prev); From 22a3a60c7775ece45e65eefa16f17f7d613413f8 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 10 Apr 2024 02:40:14 +0000 Subject: [PATCH 1153/1165] Fix special build --- src/Common/examples/parallel_aggregation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/examples/parallel_aggregation.cpp b/src/Common/examples/parallel_aggregation.cpp index 20f5f1c5224..7094690a3a8 100644 --- a/src/Common/examples/parallel_aggregation.cpp +++ b/src/Common/examples/parallel_aggregation.cpp @@ -205,7 +205,7 @@ static void aggregate4(Map & local_map, MapTwoLevel & global_map, Mutex * mutexe else { size_t hash_value = global_map.hash(*it); - size_t bucket = global_map.getBucketFromHash(hash_value); + size_t bucket = MapTwoLevel::getBucketFromHash(hash_value); if (mutexes[bucket].try_lock()) { From fc9d5ec589b978001fe43b71921fc6940c7951b9 Mon Sep 17 00:00:00 2001 From: peter279k Date: Wed, 10 Apr 2024 10:40:14 +0800 Subject: [PATCH 1154/1165] Add uptime() function usage --- .../functions/other-functions.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index e9f8bc6e547..187f248e92d 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -916,6 +916,34 @@ Returns the larger value of a and b. Returns the server’s uptime in seconds. If executed in the context of a distributed table, this function generates a normal column with values relevant to each shard. Otherwise it produces a constant value. +**Syntax** + +``` sql +uptime() +``` + +**Returned value** + +- Time value of seconds. + +Type: [UInt32](/docs/en/sql-reference/data-types/int-uint.md). + +**Example** + +Query: + +``` sql +SELECT uptime() as Uptime; +``` + +Result: + +``` response +┌─Uptime─┐ +│ 55867 │ +└────────┘ +``` + ## version() Returns the current version of ClickHouse as a string in the form of: From b96543e57e39f021cd56d3936d93b9382a5c824d Mon Sep 17 00:00:00 2001 From: peter279k Date: Wed, 10 Apr 2024 14:25:49 +0800 Subject: [PATCH 1155/1165] Add translateUTF8 function usage --- .../functions/string-replace-functions.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index c7bd16cad4a..ab39b064a8b 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -193,3 +193,33 @@ Result: ## translateUTF8 Like [translate](#translate) but assumes `s`, `from` and `to` are UTF-8 encoded strings. + +**Syntax** + +``` sql +translateUTF8(s, from, to) +``` + +**Parameters** + +- `s`: A string type [String](/docs/en/sql-reference/data-types/string.md). +- `from`: A string type [String](/docs/en/sql-reference/data-types/string.md). +- `to`: A string type [String](/docs/en/sql-reference/data-types/string.md). + +**Returned value** + +- `s`: A string type [String](/docs/en/sql-reference/data-types/string.md). + +**Examples** + +Query: + +``` sql +SELECT translateUTF8('Hello, World!', 'delor', 'DELOR') AS res; +``` + +``` response +┌─res───────────┐ +│ HELLO, WORLD! │ +└───────────────┘ +``` From 01f3d57e4be00682ed68557cec2d111ad77ef860 Mon Sep 17 00:00:00 2001 From: peter279k Date: Wed, 10 Apr 2024 12:24:07 +0800 Subject: [PATCH 1156/1165] Add upper and upperUTF8 function usage --- .../functions/string-functions.md | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 573790f7ff7..9ca77eff0e0 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -260,8 +260,36 @@ Alias: `lcase` Converts the ASCII Latin symbols in a string to uppercase. +**Syntax** + +``` sql +upper(input) +``` + Alias: `ucase` +**Parameters** + +- `input`: A string type [String](/docs/en/sql-reference/data-types/string.md). + +**Returned value** + +- A [String](/docs/en/sql-reference/data-types/string.md) data type value. + +**Examples** + +Query: + +``` sql +SELECT upper('value') as Upper; +``` + +``` response +┌─Upper─┐ +│ VALUE │ +└───────┘ +``` + ## lowerUTF8 Converts a string to lowercase, assuming that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined. @@ -278,6 +306,34 @@ Does not detect the language, e.g. for Turkish the result might not be exactly c If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point. +**Syntax** + +``` sql +upperUTF8(input) +``` + +**Parameters** + +- `input`: A string type [String](/docs/en/sql-reference/data-types/string.md). + +**Returned value** + +- A [String](/docs/en/sql-reference/data-types/string.md) data type value. + +**Example** + +Query: + +``` sql +SELECT upperUTF8('value') as Upperutf8; +``` + +``` response +┌─Upperutf8─┐ +│ VALUE │ +└───────────┘ +``` + ## isValidUTF8 Returns 1, if the set of bytes constitutes valid UTF-8-encoded text, otherwise 0. From 7774a2a313585312527db34e168e56f5405cb157 Mon Sep 17 00:00:00 2001 From: loselarry Date: Wed, 10 Apr 2024 17:21:25 +0800 Subject: [PATCH 1157/1165] chore: fix some comments Signed-off-by: loselarry --- base/poco/Foundation/src/pcre_compile.c | 2 +- docs/en/operations/settings/merge-tree-settings.md | 2 +- docs/en/operations/settings/settings.md | 2 +- src/Functions/FunctionsDecimalArithmetics.h | 2 +- src/Functions/serverConstants.cpp | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/poco/Foundation/src/pcre_compile.c b/base/poco/Foundation/src/pcre_compile.c index 3a6fafe8d56..b5f5f9a8286 100644 --- a/base/poco/Foundation/src/pcre_compile.c +++ b/base/poco/Foundation/src/pcre_compile.c @@ -4835,7 +4835,7 @@ for (;; ptr++) If the class contains characters outside the 0-255 range, a different opcode is compiled. It may optionally have a bit map for characters < 256, - but those above are are explicitly listed afterwards. A flag byte tells + but those above are explicitly listed afterwards. A flag byte tells whether the bitmap is present, and whether this is a negated class or not. In JavaScript compatibility mode, an isolated ']' causes an error. In diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 3e411a51ff4..9327d52227f 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -287,7 +287,7 @@ Default value: 0 (seconds) ## remote_fs_execute_merges_on_single_replica_time_threshold -When this setting has a value greater than than zero only a single replica starts the merge immediately if merged part on shared storage and `allow_remote_fs_zero_copy_replication` is enabled. +When this setting has a value greater than zero only a single replica starts the merge immediately if merged part on shared storage and `allow_remote_fs_zero_copy_replication` is enabled. :::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index e4e7be83f7d..3e38c22dd8f 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -5302,7 +5302,7 @@ SETTINGS(dictionary_use_async_executor=1, max_threads=8); ## storage_metadata_write_full_object_key {#storage_metadata_write_full_object_key} When set to `true` the metadata files are written with `VERSION_FULL_OBJECT_KEY` format version. With that format full object storage key names are written to the metadata files. -When set to `false` the metadata files are written with the previous format version, `VERSION_INLINE_DATA`. With that format only suffixes of object storage key names are are written to the metadata files. The prefix for all of object storage key names is set in configurations files at `storage_configuration.disks` section. +When set to `false` the metadata files are written with the previous format version, `VERSION_INLINE_DATA`. With that format only suffixes of object storage key names are written to the metadata files. The prefix for all of object storage key names is set in configurations files at `storage_configuration.disks` section. Default value: `false`. diff --git a/src/Functions/FunctionsDecimalArithmetics.h b/src/Functions/FunctionsDecimalArithmetics.h index 79e10d215a9..e26ad7362b3 100644 --- a/src/Functions/FunctionsDecimalArithmetics.h +++ b/src/Functions/FunctionsDecimalArithmetics.h @@ -280,7 +280,7 @@ public: /** At compile time, result is unknown. We only know the Scale (number of fractional digits) at runtime. Also nothing is known about size of whole part. - As in simple division/multiplication for decimals, we scale the result up, but is is explicit here and no downscale is performed. + As in simple division/multiplication for decimals, we scale the result up, but it is explicit here and no downscale is performed. It guarantees that result will have given scale and it can also be MANUALLY converted to other decimal types later. **/ if (scale > DecimalUtils::max_precision) diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp index fd8fb22455b..e7e423058f1 100644 --- a/src/Functions/serverConstants.cpp +++ b/src/Functions/serverConstants.cpp @@ -32,7 +32,7 @@ namespace #endif - /// Get the host name. Is is constant on single server, but is not constant in distributed queries. + /// Get the host name. It is constant on single server, but is not constant in distributed queries. class FunctionHostName : public FunctionConstantBase { public: From 1f48b97d23ad8dd03cbad66280db1def4c489b51 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 10 Apr 2024 09:56:54 +0000 Subject: [PATCH 1158/1165] Fix clang-tidy build --- src/Common/examples/parallel_aggregation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/examples/parallel_aggregation.cpp b/src/Common/examples/parallel_aggregation.cpp index 20f5f1c5224..7094690a3a8 100644 --- a/src/Common/examples/parallel_aggregation.cpp +++ b/src/Common/examples/parallel_aggregation.cpp @@ -205,7 +205,7 @@ static void aggregate4(Map & local_map, MapTwoLevel & global_map, Mutex * mutexe else { size_t hash_value = global_map.hash(*it); - size_t bucket = global_map.getBucketFromHash(hash_value); + size_t bucket = MapTwoLevel::getBucketFromHash(hash_value); if (mutexes[bucket].try_lock()) { From 927c5ca1fcefd3091c9a9967da2f1e1bb1affc74 Mon Sep 17 00:00:00 2001 From: peter279k Date: Wed, 10 Apr 2024 18:09:02 +0800 Subject: [PATCH 1159/1165] Update example --- .../sql-reference/functions/string-replace-functions.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index ab39b064a8b..60fe286de25 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -215,11 +215,11 @@ translateUTF8(s, from, to) Query: ``` sql -SELECT translateUTF8('Hello, World!', 'delor', 'DELOR') AS res; +SELECT translateUTF8('Münchener Straße', 'üß', 'us') AS res; ``` ``` response -┌─res───────────┐ -│ HELLO, WORLD! │ -└───────────────┘ +┌─res──────────────┐ +│ Munchener Strase │ +└──────────────────┘ ``` From 06bbf97cc057fc872d0b0749b0d349d21bbb0f72 Mon Sep 17 00:00:00 2001 From: peter279k Date: Wed, 10 Apr 2024 18:12:29 +0800 Subject: [PATCH 1160/1165] Update example --- docs/en/sql-reference/functions/string-functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 9ca77eff0e0..d4df3e0479a 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -325,12 +325,12 @@ upperUTF8(input) Query: ``` sql -SELECT upperUTF8('value') as Upperutf8; +SELECT upperUTF8('München') as Upperutf8; ``` ``` response ┌─Upperutf8─┐ -│ VALUE │ +│ MÜNCHEN │ └───────────┘ ``` From 444ad3bf62f2099de1978bcddc0413255fe8ac93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 10 Apr 2024 12:12:53 +0200 Subject: [PATCH 1161/1165] Fix random clang tidy warning --- src/Common/examples/encrypt_decrypt.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/examples/encrypt_decrypt.cpp b/src/Common/examples/encrypt_decrypt.cpp index c7f949195c8..18d4e50be8a 100644 --- a/src/Common/examples/encrypt_decrypt.cpp +++ b/src/Common/examples/encrypt_decrypt.cpp @@ -35,9 +35,9 @@ int main(int argc, char ** argv) DB::CompressionCodecEncrypted::Configuration::instance().load(*loaded_config.configuration, "encryption_codecs"); if (action == "-e") - std::cout << processor.encryptValue(codec_name, value) << std::endl; + std::cout << DB::ConfigProcessor::encryptValue(codec_name, value) << std::endl; else if (action == "-d") - std::cout << processor.decryptValue(codec_name, value) << std::endl; + std::cout << DB::ConfigProcessor::decryptValue(codec_name, value) << std::endl; else std::cerr << "Unknown action: " << action << std::endl; } From e76aefac9c574ab11fc715109728cc8612df61e0 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 10 Apr 2024 11:55:38 +0000 Subject: [PATCH 1162/1165] Removing users.xml --- tests/queries/0_stateless/users.xml | 110 ---------------------------- 1 file changed, 110 deletions(-) delete mode 100644 tests/queries/0_stateless/users.xml diff --git a/tests/queries/0_stateless/users.xml b/tests/queries/0_stateless/users.xml deleted file mode 100644 index a199435b42f..00000000000 --- a/tests/queries/0_stateless/users.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - - - - - - - - - 1 - - - - - - - - - c64c5e4e53ea1a9f1427d2713b3a22bbebe8940bc807adaf654744b1568c70ab - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - From 715e6d90d7c1bbcb9d4f179a7faa6da716bff776 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 10 Apr 2024 11:56:32 +0000 Subject: [PATCH 1163/1165] Removing 02203_shebang.bak --- tests/queries/0_stateless/02203_shebang.bak | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 tests/queries/0_stateless/02203_shebang.bak diff --git a/tests/queries/0_stateless/02203_shebang.bak b/tests/queries/0_stateless/02203_shebang.bak deleted file mode 100755 index 07686d1aab4..00000000000 --- a/tests/queries/0_stateless/02203_shebang.bak +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/clickhouse-local --queries-file - -SELECT 1; From 7bf4976198b38f0a472d56ba7138d877b68934eb Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 10 Apr 2024 12:57:48 +0000 Subject: [PATCH 1164/1165] impl --- tests/queries/0_stateless/01592_long_window_functions1.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/01592_long_window_functions1.sql b/tests/queries/0_stateless/01592_long_window_functions1.sql index c63c651fb0b..d2d32e24eaa 100644 --- a/tests/queries/0_stateless/01592_long_window_functions1.sql +++ b/tests/queries/0_stateless/01592_long_window_functions1.sql @@ -1,5 +1,8 @@ -- Tags: long +-- test became more than an order of magnitude slower with max_bytes_before_external_sort=1 +set max_bytes_before_external_sort = 0; + drop table if exists stack; set max_insert_threads = 4; From 4ed518d6ad83c1e69326e527882de57f127e6a8b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 10 Apr 2024 14:18:47 +0000 Subject: [PATCH 1165/1165] Docs: Update date_add --- docs/en/sql-reference/functions/date-time-functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 4c419004344..3bb9d4e7dbe 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1906,7 +1906,7 @@ Aliases: `dateAdd`, `DATE_ADD`. **Arguments** -- `unit` — The type of interval to add. [String](../../sql-reference/data-types/string.md). +- `unit` — The type of interval to add. Note: This is not a [String](../../sql-reference/data-types/string.md) and must therefore not be quoted. Possible values: - `second` @@ -1961,7 +1961,7 @@ Aliases: `dateSub`, `DATE_SUB`. **Arguments** -- `unit` — The type of interval to subtract. Note: The unit should be unquoted. +- `unit` — The type of interval to subtract. Note: This is not a [String](../../sql-reference/data-types/string.md) and must therefore not be quoted. Possible values: