From 18c2ac2e1ed4fa5519f5a37771da1d2cc6134cdf Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Wed, 13 Oct 2021 15:43:52 +0000 Subject: [PATCH 01/39] add parallel reading from infile in client --- src/Client/ClientBase.cpp | 41 ++++++++++++++++--- src/Client/ClientBase.h | 3 ++ ...048_parallel_reading_from_infile.reference | 1 + .../02048_parallel_reading_from_infile.sh | 23 +++++++++++ 4 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/02048_parallel_reading_from_infile.reference create mode 100755 tests/queries/0_stateless/02048_parallel_reading_from_infile.sh diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index cde5a5f9977..cbe09f62ae9 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -9,6 +9,7 @@ #include #include #include +#include "Common/getNumberOfPhysicalCPUCores.h" #if !defined(ARCADIA_BUILD) # include @@ -773,7 +774,8 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des /// Get name of this file (path to file) const auto & in_file_node = parsed_insert_query->infile->as(); const auto in_file = in_file_node.value.safeGet(); - + /// Get name of table + const auto table_name = parsed_insert_query->table_id.getTableName(); std::string compression_method; /// Compression method can be specified in query if (parsed_insert_query->compression) @@ -782,13 +784,35 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des compression_method = compression_method_node.value.safeGet(); } - /// Otherwise, it will be detected from file name automatically (by chooseCompressionMethod) - /// Buffer for reading from file is created and wrapped with appropriate compression method - auto in_buffer = wrapReadBufferWithCompressionMethod(std::make_unique(in_file), chooseCompressionMethod(in_file, compression_method)); - + /// Create temporary storage file, to support globs and parallel reading + StorageFile::CommonArguments args{ + WithContext(global_context), + StorageID("_from_infile", table_name), + parsed_insert_query->format, + std::nullopt /*format settings*/, + compression_method, + columns_description, + ConstraintsDescription{}, + String{}, + }; + StoragePtr storage = StorageFile::create(in_file, global_context->getUserFilesPath(), args); + storage->startup(); + SelectQueryInfo query_info; + try { - sendDataFrom(*in_buffer, sample, columns_description, parsed_query); + sendDataFromPipe( + storage->read( + sample.getNames(), + storage->getInMemoryMetadataPtr(), + query_info, + global_context, + {}, + DEFAULT_BLOCK_SIZE, + getNumberOfPhysicalCPUCores() + ), + parsed_query + ); } catch (Exception & e) { @@ -862,6 +886,11 @@ void ClientBase::sendDataFrom(ReadBuffer & buf, Block & sample, const ColumnsDes }); } + sendDataFromPipe(std::move(pipe), parsed_query); +} + +void ClientBase::sendDataFromPipe(Pipe&& pipe, ASTPtr parsed_query) +{ QueryPipeline pipeline(std::move(pipe)); PullingAsyncPipelineExecutor executor(pipeline); diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index bf9e8fdfe47..f78e5320422 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include namespace po = boost::program_options; @@ -113,6 +115,7 @@ private: void sendData(Block & sample, const ColumnsDescription & columns_description, ASTPtr parsed_query); void sendDataFrom(ReadBuffer & buf, Block & sample, const ColumnsDescription & columns_description, ASTPtr parsed_query); + void sendDataFromPipe(Pipe&& pipe, ASTPtr parsed_query); void sendExternalTables(ASTPtr parsed_query); void initBlockOutputStream(const Block & block, ASTPtr parsed_query); diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference new file mode 100644 index 00000000000..00750edc07d --- /dev/null +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference @@ -0,0 +1 @@ +3 diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh new file mode 100755 index 00000000000..ee408fc2abb --- /dev/null +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e + +[ -e "${CLICKHOUSE_TMP}"/test_infile_parallel.gz ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel.gz +[ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel +[ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel_1 +[ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel_2 + +echo "Hello" > "${CLICKHOUSE_TMP}"/test_infile_parallel +echo "Hello" > "${CLICKHOUSE_TMP}"/test_infile_parallel_1 +echo "Hello" > "${CLICKHOUSE_TMP}"/test_infile_parallel_2 + +gzip "${CLICKHOUSE_TMP}"/test_infile_parallel + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile_parallel;" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile_parallel (word String) ENGINE=Memory();" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT CSV;" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel;" From 2384108ea5084caf47199648a25874a1d4ab381d Mon Sep 17 00:00:00 2001 From: Filatenkov Artur <58165623+FArthur-cmd@users.noreply.github.com> Date: Fri, 22 Oct 2021 15:32:24 +0300 Subject: [PATCH 02/39] remove trailing whitespaces --- src/Client/ClientBase.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 183236cee6b..4c47142dd93 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -871,7 +871,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des StoragePtr storage = StorageFile::create(in_file, global_context->getUserFilesPath(), args); storage->startup(); SelectQueryInfo query_info; - + try { sendDataFromPipe( @@ -883,7 +883,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des {}, DEFAULT_BLOCK_SIZE, getNumberOfPhysicalCPUCores() - ), + ), parsed_query ); } From 20c5e9d3217926911a0891ec9e9d277a0196db4a Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Tue, 26 Oct 2021 15:16:58 +0000 Subject: [PATCH 03/39] refactoring, adding tests for local --- src/Client/ClientBase.cpp | 22 ++++++++++++++----- src/Client/ClientBase.h | 2 +- ...048_parallel_reading_from_infile.reference | 1 + .../02048_parallel_reading_from_infile.sh | 7 ++++++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 4c47142dd93..af9e1ae9889 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -9,11 +9,14 @@ #include #include #include +#include "Common/Exception.h" #include "Common/getNumberOfPhysicalCPUCores.h" +#include "Common/tests/gtest_global_context.h" #include "Columns/ColumnString.h" #include "Columns/ColumnsNumber.h" #include "Core/Block.h" #include "Core/Protocol.h" +#include "Formats/FormatFactory.h" #if !defined(ARCADIA_BUILD) # include @@ -826,6 +829,13 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_description, ASTPtr parsed_query) { + /// Get columns description from variable or (if it was empty) create it from sample. + auto columns_description_for_query = columns_description.empty() ? ColumnsDescription(sample.getNamesAndTypesList()) : columns_description; + if (columns_description_for_query.empty()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column description is empty and it can't be built from sample from table. Cannot execute query."); + } + /// If INSERT data must be sent. auto * parsed_insert_query = parsed_query->as(); if (!parsed_insert_query) @@ -860,11 +870,11 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des /// Create temporary storage file, to support globs and parallel reading StorageFile::CommonArguments args{ WithContext(global_context), - StorageID("_from_infile", table_name), + parsed_insert_query->table_id, parsed_insert_query->format, - std::nullopt /*format settings*/, + getFormatSettings(global_context), compression_method, - columns_description, + columns_description_for_query, ConstraintsDescription{}, String{}, }; @@ -881,7 +891,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des query_info, global_context, {}, - DEFAULT_BLOCK_SIZE, + global_context->getSettingsRef().max_block_size, getNumberOfPhysicalCPUCores() ), parsed_query @@ -899,7 +909,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des ReadBufferFromMemory data_in(parsed_insert_query->data, parsed_insert_query->end - parsed_insert_query->data); try { - sendDataFrom(data_in, sample, columns_description, parsed_query); + sendDataFrom(data_in, sample, columns_description_for_query, parsed_query); } catch (Exception & e) { @@ -924,7 +934,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des /// Send data read from stdin. try { - sendDataFrom(std_in, sample, columns_description, parsed_query); + sendDataFrom(std_in, sample, columns_description_for_query, parsed_query); } catch (Exception & e) { diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 3fdeac6b943..7f4373dddb6 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -122,7 +122,7 @@ private: void sendData(Block & sample, const ColumnsDescription & columns_description, ASTPtr parsed_query); void sendDataFrom(ReadBuffer & buf, Block & sample, const ColumnsDescription & columns_description, ASTPtr parsed_query); - void sendDataFromPipe(Pipe&& pipe, ASTPtr parsed_query); + void sendDataFromPipe(Pipe && pipe, ASTPtr parsed_query); void sendExternalTables(ASTPtr parsed_query); void initBlockOutputStream(const Block & block, ASTPtr parsed_query); diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference index 00750edc07d..a5c8806279f 100644 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference @@ -1 +1,2 @@ 3 +3 diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh index ee408fc2abb..d300ebe8b99 100755 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh @@ -21,3 +21,10 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile_parallel;" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile_parallel (word String) ENGINE=Memory();" ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT CSV;" ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel;" + +${CLICKHOUSE_LOCAL} --multiquery < Date: Tue, 26 Oct 2021 22:05:34 +0300 Subject: [PATCH 04/39] hsts-max-age --- .../server-configuration-parameters/settings.md | 2 +- .../server-configuration-parameters/settings.md | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index cdf49678570..963393187b2 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -366,7 +366,7 @@ Opens `https://tabix.io/` when accessing `http://localhost: http_port`.
]]> ``` -## hsts_max_age +## hsts_max_age {#hsts-max-age} Expired time for HSTS in seconds. The default value is 0 means clickhouse disabled HSTS. If you set a positive number, the HSTS will be enabled and the max-age is the number you set. diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 2d4ca1d3dcd..188019fc5c7 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -368,6 +368,16 @@ ClickHouse проверяет условия для `min_part_size` и `min_part ``` +## hsts_max_age {#hsts-max-age} + +Срок действия HSTS в секундах. Значение по умолчанию `0` (HSTS выключен). Для включения HSTS задайте положительное число. Срок действия HSTS будет равен введенному числу. + +**Пример** + +```xml +600000 +``` + ## include_from {#server_configuration_parameters-include_from} Путь к файлу с подстановками. From 019971afabc3938d01fb30a7f27b4d3431212552 Mon Sep 17 00:00:00 2001 From: andrc1901 <92211164+andrc1901@users.noreply.github.com> Date: Tue, 26 Oct 2021 22:24:04 +0300 Subject: [PATCH 05/39] Update settings.md --- docs/ru/operations/server-configuration-parameters/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 188019fc5c7..36cc4fd4f09 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -370,7 +370,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part ## hsts_max_age {#hsts-max-age} -Срок действия HSTS в секундах. Значение по умолчанию `0` (HSTS выключен). Для включения HSTS задайте положительное число. Срок действия HSTS будет равен введенному числу. +Срок действия HSTS в секундах. Значение по умолчанию `0` (HSTS выключен). Для включения HSTS задайте положительное число. Срок действия HSTS будет равен введенному числу. **Пример** From 08c41742755aef951ad61746c03805a3e9151647 Mon Sep 17 00:00:00 2001 From: Andrei Ch Date: Wed, 27 Oct 2021 00:15:40 +0300 Subject: [PATCH 06/39] sql-support-in-russian --- docs/ru/introduction/distinctive-features.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/ru/introduction/distinctive-features.md b/docs/ru/introduction/distinctive-features.md index dedb1412dbf..b330b2a0b41 100644 --- a/docs/ru/introduction/distinctive-features.md +++ b/docs/ru/introduction/distinctive-features.md @@ -30,11 +30,13 @@ toc_title: "Отличительные возможности ClickHouse" Почти все перечисленные ранее столбцовые СУБД не поддерживают распределённую обработку запроса. В ClickHouse данные могут быть расположены на разных шардах. Каждый шард может представлять собой группу реплик, которые используются для отказоустойчивости. Запрос будет выполнен на всех шардах параллельно. Это делается прозрачно для пользователя. -## Поддержка SQL {#podderzhka-sql} +## Поддержка SQL {#sql-support} -ClickHouse поддерживает декларативный язык запросов на основе SQL и во многих случаях совпадающий с SQL стандартом. -Поддерживаются GROUP BY, ORDER BY, подзапросы в секциях FROM, IN, JOIN, а также скалярные подзапросы. -Зависимые подзапросы и оконные функции не поддерживаются. +ClickHouse поддерживает [декларативный язык запросов на основе SQL](../sql-reference/statements/select/group-by.md) и во [многих случаях](../sql-reference/ansi.md) совпадающий с SQL стандартом. + +Поддерживаются [GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), подзапросы в секциях [FROM](../sql-reference/statements/select/from.md), [IN](../sql-reference/operators/in.md), [JOIN](../sql-reference/statements/select/join.md), [функции window](../sql-reference/window-functions/index.md), а также скалярные подзапросы. + +Зависимые подзапросы не поддерживаются, но могут стать доступными в будущем. ## Векторный движок {#vektornyi-dvizhok} From f2854466d02bfcfc3652b8387c7df92591380897 Mon Sep 17 00:00:00 2001 From: Andrei Ch Date: Wed, 27 Oct 2021 00:47:49 +0300 Subject: [PATCH 07/39] Update data-skipping-indices --- .../ru/operations/system-tables/data_skipping_indices.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/ru/operations/system-tables/data_skipping_indices.md b/docs/ru/operations/system-tables/data_skipping_indices.md index 39e13ed1d5a..86ae6576ec9 100644 --- a/docs/ru/operations/system-tables/data_skipping_indices.md +++ b/docs/ru/operations/system-tables/data_skipping_indices.md @@ -10,6 +10,9 @@ - `type` ([String](../../sql-reference/data-types/string.md)) — тип индекса. - `expr` ([String](../../sql-reference/data-types/string.md)) — выражение, используемое для вычисления индекса. - `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — количество гранул в блоке данных. +- `data_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер сжатых данных в байтах. +- `data_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер распакованных данных в байтах. +- `marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер меток в байтах. **Пример** @@ -26,6 +29,9 @@ name: clicks_idx type: minmax expr: clicks granularity: 1 +data_compressed_bytes: 58 +data_uncompressed_bytes: 6 +marks: 48 Row 2: ────── @@ -35,4 +41,7 @@ name: contacts_null_idx type: minmax expr: assumeNotNull(contacts_null) granularity: 1 +data_compressed_bytes: 58 +data_uncompressed_bytes: 6 +marks: 48 ``` From 2c98eb198f55ff7ac1fcf8f5e64bfc60967441e2 Mon Sep 17 00:00:00 2001 From: andrc1901 <92211164+andrc1901@users.noreply.github.com> Date: Wed, 27 Oct 2021 00:57:05 +0300 Subject: [PATCH 08/39] Update data_skipping_indices.md --- docs/ru/operations/system-tables/data_skipping_indices.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/data_skipping_indices.md b/docs/ru/operations/system-tables/data_skipping_indices.md index 86ae6576ec9..07df40d2c21 100644 --- a/docs/ru/operations/system-tables/data_skipping_indices.md +++ b/docs/ru/operations/system-tables/data_skipping_indices.md @@ -11,7 +11,7 @@ - `expr` ([String](../../sql-reference/data-types/string.md)) — выражение, используемое для вычисления индекса. - `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — количество гранул в блоке данных. - `data_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер сжатых данных в байтах. -- `data_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер распакованных данных в байтах. +- `data_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер несжатых данных в байтах. - `marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер меток в байтах. **Пример** From ba0f5fd199d5d8c62a3509e84f069630be1d4899 Mon Sep 17 00:00:00 2001 From: Andrei Ch Date: Wed, 27 Oct 2021 01:07:27 +0300 Subject: [PATCH 09/39] update parts.md --- docs/ru/operations/system-tables/parts.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/ru/operations/system-tables/parts.md b/docs/ru/operations/system-tables/parts.md index c73e1566a95..c9ae7aaff76 100644 --- a/docs/ru/operations/system-tables/parts.md +++ b/docs/ru/operations/system-tables/parts.md @@ -38,6 +38,12 @@ - `marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – размер файла с засечками. +- `secondary_indices_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – общий размер сжатых данных для вторичных индексов в куске данных. Вспомогательные файлы (например, файлы с метками) не включены. + +- `secondary_indices_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – общий размер несжатых данных для вторичных индексов в куске данных. Вспомогательные файлы (например, файлы с метками) не включены. + +- `secondary_indices_marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – размер файла с метками для вторичных индексов. + - `modification_time` ([DateTime](../../sql-reference/data-types/datetime.md)) – время модификации директории с куском данных. Обычно соответствует времени создания куска. - `remove_time` ([DateTime](../../sql-reference/data-types/datetime.md)) – время, когда кусок стал неактивным. @@ -119,6 +125,9 @@ rows: 6 bytes_on_disk: 310 data_compressed_bytes: 157 data_uncompressed_bytes: 91 +secondary_indices_compressed_bytes: 58 +secondary_indices_uncompressed_bytes: 6 +secondary_indices_marks_bytes: 48 marks_bytes: 144 modification_time: 2020-06-18 13:01:49 remove_time: 0000-00-00 00:00:00 From c397967135bf3e5b60b7d7fca021d26741a05981 Mon Sep 17 00:00:00 2001 From: andrc1901 <92211164+andrc1901@users.noreply.github.com> Date: Wed, 27 Oct 2021 01:12:06 +0300 Subject: [PATCH 10/39] Update parts.md --- docs/ru/operations/system-tables/parts.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/operations/system-tables/parts.md b/docs/ru/operations/system-tables/parts.md index c9ae7aaff76..dd3d945daf5 100644 --- a/docs/ru/operations/system-tables/parts.md +++ b/docs/ru/operations/system-tables/parts.md @@ -38,11 +38,11 @@ - `marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – размер файла с засечками. -- `secondary_indices_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – общий размер сжатых данных для вторичных индексов в куске данных. Вспомогательные файлы (например, файлы с метками) не включены. +- `secondary_indices_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – общий размер сжатых данных для вторичных индексов в куске данных. Вспомогательные файлы (например, файлы с засечками) не включены. -- `secondary_indices_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – общий размер несжатых данных для вторичных индексов в куске данных. Вспомогательные файлы (например, файлы с метками) не включены. +- `secondary_indices_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – общий размер несжатых данных для вторичных индексов в куске данных. Вспомогательные файлы (например, файлы с засечками) не включены. -- `secondary_indices_marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – размер файла с метками для вторичных индексов. +- `secondary_indices_marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – размер файла с засечками для вторичных индексов. - `modification_time` ([DateTime](../../sql-reference/data-types/datetime.md)) – время модификации директории с куском данных. Обычно соответствует времени создания куска. From 72685d0c7848c568bd7a5d6fcd7b2da846f38b87 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Thu, 28 Oct 2021 18:46:51 +0000 Subject: [PATCH 11/39] refactoring and test improvement --- src/Client/ClientBase.cpp | 3 ++- ...048_parallel_reading_from_infile.reference | 2 ++ .../02048_parallel_reading_from_infile.sh | 24 +++++++++++-------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index af9e1ae9889..aab3b4b344e 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -12,6 +12,7 @@ #include "Common/Exception.h" #include "Common/getNumberOfPhysicalCPUCores.h" #include "Common/tests/gtest_global_context.h" +#include "Common/typeid_cast.h" #include "Columns/ColumnString.h" #include "Columns/ColumnsNumber.h" #include "Core/Block.h" @@ -833,7 +834,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des auto columns_description_for_query = columns_description.empty() ? ColumnsDescription(sample.getNamesAndTypesList()) : columns_description; if (columns_description_for_query.empty()) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column description is empty and it can't be built from sample from table. Cannot execute query."); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Column description is empty and it can't be built from sample from table. Cannot execute query."); } /// If INSERT data must be sent. diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference index a5c8806279f..e356ab2095d 100644 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference @@ -1,2 +1,4 @@ +4 +4 3 3 diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh index d300ebe8b99..4c5d586280e 100755 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh @@ -11,20 +11,24 @@ set -e [ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel_1 [ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel_2 -echo "Hello" > "${CLICKHOUSE_TMP}"/test_infile_parallel -echo "Hello" > "${CLICKHOUSE_TMP}"/test_infile_parallel_1 -echo "Hello" > "${CLICKHOUSE_TMP}"/test_infile_parallel_2 +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile_parallel;" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile_parallel (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory();" +${CLICKHOUSE_CLIENT} --query "SET input_format_tsv_enum_as_number = 1;" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel VALUES (102, 2), (103, 1);" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE 'test_infile_parallel' FORMAT TSV;" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE 'test_infile_parallel_1' FORMAT TSV;" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE 'test_infile_parallel_2' FORMAT TSV;" gzip "${CLICKHOUSE_TMP}"/test_infile_parallel -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile_parallel;" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile_parallel (word String) ENGINE=Memory();" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT CSV;" -${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel;" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT TSV;" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='first';" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='second';" ${CLICKHOUSE_LOCAL} --multiquery < Date: Thu, 28 Oct 2021 21:52:44 +0300 Subject: [PATCH 12/39] Update docs/ru/operations/system-tables/data_skipping_indices.md Co-authored-by: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> --- docs/ru/operations/system-tables/data_skipping_indices.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/data_skipping_indices.md b/docs/ru/operations/system-tables/data_skipping_indices.md index 07df40d2c21..d57d62cf08b 100644 --- a/docs/ru/operations/system-tables/data_skipping_indices.md +++ b/docs/ru/operations/system-tables/data_skipping_indices.md @@ -12,7 +12,7 @@ - `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — количество гранул в блоке данных. - `data_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер сжатых данных в байтах. - `data_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер несжатых данных в байтах. -- `marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер меток в байтах. +- `marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — размер засечек в байтах. **Пример** From 89d8fb76222d176756e06ef3aacd74ecb3a49c59 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Thu, 28 Oct 2021 21:23:58 +0000 Subject: [PATCH 13/39] correct test --- src/Client/ClientBase.cpp | 1 + .../0_stateless/02048_parallel_reading_from_infile.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index aab3b4b344e..8640af0af29 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -77,6 +77,7 @@ namespace ErrorCodes extern const int INVALID_USAGE_OF_INPUT; extern const int CANNOT_SET_SIGNAL_HANDLER; extern const int UNRECOGNIZED_ARGUMENTS; + extern const int LOGICAL_ERROR; } } diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh index 4c5d586280e..9c957305346 100755 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh @@ -15,9 +15,9 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile_parallel;" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile_parallel (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory();" ${CLICKHOUSE_CLIENT} --query "SET input_format_tsv_enum_as_number = 1;" ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel VALUES (102, 2), (103, 1);" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE 'test_infile_parallel' FORMAT TSV;" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE 'test_infile_parallel_1' FORMAT TSV;" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE 'test_infile_parallel_2' FORMAT TSV;" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE '${CLICKHOUSE_TMP}/test_infile_parallel' FORMAT TSV;" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE '${CLICKHOUSE_TMP}/test_infile_parallel_1' FORMAT TSV;" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE '${CLICKHOUSE_TMP}/test_infile_parallel_2' FORMAT TSV;" gzip "${CLICKHOUSE_TMP}"/test_infile_parallel From 07bab5193f20e4bf7b8dca91bb25bb90ffb86c4b Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 29 Oct 2021 15:04:08 +0300 Subject: [PATCH 14/39] Delayed interactive --- programs/client/Client.cpp | 70 +++++++++++-------- programs/client/Client.h | 1 + programs/local/LocalServer.cpp | 34 +++++++-- programs/local/LocalServer.h | 1 + src/Client/ClientBase.cpp | 2 + src/Client/ClientBase.h | 2 + ...clickhouse_client_with_queries_file.expect | 30 ++++++++ ...ckhouse_client_with_queries_file.reference | 0 .../02112_delayed_clickhouse_local.expect | 24 +++++++ ..._delayed_clickhouse_local.expect.reference | 0 .../02112_delayed_clickhouse_local.reference | 0 ..._clickhouse_local_with_queries_file.expect | 30 ++++++++ ...ickhouse_local_with_queries_file.reference | 0 .../0_stateless/helpers/02112_clean.sh | 6 ++ .../0_stateless/helpers/02112_prepare.sh | 7 ++ 15 files changed, 169 insertions(+), 38 deletions(-) create mode 100755 tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect create mode 100644 tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.reference create mode 100755 tests/queries/0_stateless/02112_delayed_clickhouse_local.expect create mode 100644 tests/queries/0_stateless/02112_delayed_clickhouse_local.expect.reference create mode 100644 tests/queries/0_stateless/02112_delayed_clickhouse_local.reference create mode 100755 tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect create mode 100644 tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.reference create mode 100755 tests/queries/0_stateless/helpers/02112_clean.sh create mode 100755 tests/queries/0_stateless/helpers/02112_prepare.sh diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 4951106f595..8985196bb0e 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -403,6 +403,33 @@ void Client::initialize(Poco::Util::Application & self) } +void Client::prepareForInteractive() +{ + clearTerminal(); + showClientVersion(); + + /// Load Warnings at the beginning of connection + if (!config().has("no-warnings")) + { + try + { + std::vector messages = loadWarningMessages(); + if (!messages.empty()) + { + std::cout << "Warnings:" << std::endl; + for (const auto & message : messages) + std::cout << " * " << message << std::endl; + std::cout << std::endl; + } + } + catch (...) + { + /// Ignore exception + } + } +} + + int Client::main(const std::vector & /*args*/) try { @@ -429,36 +456,11 @@ try processConfig(); - if (is_interactive) - { - clearTerminal(); - showClientVersion(); - } - connect(); - if (is_interactive) + if (is_interactive && !delayed_interactive) { - /// Load Warnings at the beginning of connection - if (!config().has("no-warnings")) - { - try - { - std::vector messages = loadWarningMessages(); - if (!messages.empty()) - { - std::cout << "Warnings:" << std::endl; - for (const auto & message : messages) - std::cout << " * " << message << std::endl; - std::cout << std::endl; - } - } - catch (...) - { - /// Ignore exception - } - } - + prepareForInteractive(); runInteractive(); } else @@ -482,6 +484,12 @@ try // case so that at least we don't lose an error. return -1; } + + if (delayed_interactive) + { + prepareForInteractive(); + runInteractive(); + } } return 0; @@ -1161,11 +1169,11 @@ void Client::processConfig() /// - stdin is not a terminal. In this case queries are read from it. /// - -qf (--queries-file) command line option is present. /// The value of the option is used as file with query (or of multiple queries) to execute. - if (stdin_is_a_tty && !config().has("query") && queries_files.empty()) - { - if (config().has("query") && config().has("queries-file")) - throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS); + delayed_interactive = config().has("interactive") && (config().has("query") || config().has("queries-file")); + if (stdin_is_a_tty + && (delayed_interactive || (!config().has("query") && queries_files.empty()))) + { is_interactive = true; } else diff --git a/programs/client/Client.h b/programs/client/Client.h index 2def74ef3fc..b146134bc94 100644 --- a/programs/client/Client.h +++ b/programs/client/Client.h @@ -20,6 +20,7 @@ protected: bool processWithFuzzing(const String & full_query) override; void connect() override; + void prepareForInteractive() override; void processError(const String & query) const override; String getName() const override { return "client"; } diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 2a2fe4a78c8..c4e4c10777c 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -412,6 +412,14 @@ void LocalServer::connect() } +void LocalServer::prepareForInteractive() +{ + clearTerminal(); + showClientVersion(); + std::cerr << std::endl; +} + + int LocalServer::main(const std::vector & /*args*/) try { @@ -422,7 +430,10 @@ try std::cout << std::fixed << std::setprecision(3); std::cerr << std::fixed << std::setprecision(3); - is_interactive = stdin_is_a_tty && !config().has("query") && !config().has("table-structure") && queries_files.empty(); + is_interactive = stdin_is_a_tty + && (config().hasOption("interactive") + || (!config().has("query") && !config().has("table-structure") && queries_files.empty())); + if (!is_interactive) { /// We will terminate process on error @@ -443,17 +454,20 @@ try applyCmdSettings(global_context); connect(); - if (is_interactive) + if (is_interactive && !delayed_interactive) { - clearTerminal(); - showClientVersion(); - std::cerr << std::endl; - + prepareForInteractive(); runInteractive(); } else { runNonInteractive(); + + if (delayed_interactive) + { + prepareForInteractive(); + runInteractive(); + } } cleanup(); @@ -478,7 +492,8 @@ catch (...) void LocalServer::processConfig() { - if (is_interactive) + delayed_interactive = config().has("interactive") && (config().has("query") || config().has("queries-file")); + if (is_interactive && !delayed_interactive) { if (config().has("query") && config().has("queries-file")) throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS); @@ -490,6 +505,11 @@ void LocalServer::processConfig() } else { + if (delayed_interactive) + { + load_suggestions = true; + } + need_render_progress = config().getBool("progress", false); echo_queries = config().hasOption("echo") || config().hasOption("verbose"); ignore_error = config().getBool("ignore-error", false); diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index ce0df06c86a..e87e6bd9a0d 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -34,6 +34,7 @@ protected: bool executeMultiQuery(const String & all_queries_text) override; void connect() override; + void prepareForInteractive() override; void processError(const String & query) const override; String getName() const override { return "local"; } diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index ed90c777e20..de76a7aab2c 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1612,6 +1612,8 @@ void ClientBase::init(int argc, char ** argv) ("hardware-utilization", "print hardware utilization information in progress bar") ("print-profile-events", po::value(&profile_events.print)->zero_tokens(), "Printing ProfileEvents packets") ("profile-events-delay-ms", po::value()->default_value(profile_events.delay_ms), "Delay between printing `ProfileEvents` packets (-1 - print only totals, 0 - print every single packet)") + + ("interactive", "Process queries-file or --query query and start interactive mode") ; addOptions(options_description); diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index f7b418ad13a..b37613f9910 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -57,6 +57,7 @@ protected: virtual bool executeMultiQuery(const String & all_queries_text) = 0; virtual void connect() = 0; + virtual void prepareForInteractive() = 0; virtual void processError(const String & query) const = 0; virtual String getName() const = 0; @@ -138,6 +139,7 @@ private: protected: bool is_interactive = false; /// Use either interactive line editing interface or batch mode. bool is_multiquery = false; + bool delayed_interactive = false; bool echo_queries = false; /// Print queries before execution in batch mode. bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode. diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect new file mode 100755 index 00000000000..245253ac205 --- /dev/null +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect @@ -0,0 +1,30 @@ +#!/usr/bin/expect -f +# Tags: no-parallel, no-fasttest + +log_user 0 +set timeout 20 +match_max 100000 + +# A default timeout action is to fail +expect_after { + timeout { + exit 1 + } +} + + +spawn bash -c "\$CLICKHOUSE_TESTS_DIR/helpers/02112_prepare.sh" + +set basedir [file dirname $argv0] +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT --disable_suggestion --interactive --queries-file \$CURDIR/file_02112" +expect ":) " + +send -- "select * from t format TSV\r" +expect "1" +expect ":) " + +send -- "exit\r" +expect eof + +spawn bash -c "\$CLICKHOUSE_TESTS_DIR/helpers/02112_clean.sh" + diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.reference b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect new file mode 100755 index 00000000000..6bb9140ee35 --- /dev/null +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect @@ -0,0 +1,24 @@ +#!/usr/bin/expect -f +# Tags: no-unbundled, no-fasttest + +log_user 0 +set timeout 20 +match_max 100000 + +# A default timeout action is to fail +expect_after { + timeout { + exit 1 + } +} + +set basedir [file dirname $argv0] +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --disable_suggestion --interactive --query 'create table t(i Int32) engine=Memory; insert into t select 1'" +expect ":) " + +send -- "select * from t format TSV\r" +expect "1" +expect ":) " + +send -- "exit\r" +expect eof diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect.reference b/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local.reference b/tests/queries/0_stateless/02112_delayed_clickhouse_local.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect new file mode 100755 index 00000000000..774d8d63714 --- /dev/null +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect @@ -0,0 +1,30 @@ +#!/usr/bin/expect -f +# Tags: no-parallel, no-fasttest + +log_user 0 +set timeout 20 +match_max 100000 + +# A default timeout action is to fail +expect_after { + timeout { + exit 1 + } +} + + +spawn bash -c "\$CLICKHOUSE_TESTS_DIR/helpers/02112_prepare.sh" + +set basedir [file dirname $argv0] +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --disable_suggestion --interactive --queries-file \$CURDIR/file_02112" +expect ":) " + +send -- "select * from t format TSV\r" +expect "1" +expect ":) " + +send -- "exit\r" +expect eof + +spawn bash -c "\$CLICKHOUSE_TESTS_DIR/helpers/02112_clean.sh" + diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.reference b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/helpers/02112_clean.sh b/tests/queries/0_stateless/helpers/02112_clean.sh new file mode 100755 index 00000000000..910c0709955 --- /dev/null +++ b/tests/queries/0_stateless/helpers/02112_clean.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +FILE=${CURDIR}/file_02112 +if [ -f $FILE ]; then + rm $FILE +fi diff --git a/tests/queries/0_stateless/helpers/02112_prepare.sh b/tests/queries/0_stateless/helpers/02112_prepare.sh new file mode 100755 index 00000000000..1f371789f86 --- /dev/null +++ b/tests/queries/0_stateless/helpers/02112_prepare.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +FILE=${CURDIR}/file_02112 +if [ -f $FILE ]; then + rm $FILE +fi +echo "drop table if exists t;create table t(i Int32) engine=Memory; insert into t select 1" >> $FILE From 922614796d0bffa447faee296fe790a0102c0c81 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 1 Nov 2021 10:22:43 +0200 Subject: [PATCH 15/39] Update RabbitMQ docs Add description for new setting rabbitmq_queue_settings_list --- docs/ru/engines/table-engines/integrations/rabbitmq.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ru/engines/table-engines/integrations/rabbitmq.md b/docs/ru/engines/table-engines/integrations/rabbitmq.md index ef8a58c4c82..77d69b4d0cc 100644 --- a/docs/ru/engines/table-engines/integrations/rabbitmq.md +++ b/docs/ru/engines/table-engines/integrations/rabbitmq.md @@ -30,6 +30,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [rabbitmq_skip_broken_messages = N,] [rabbitmq_max_block_size = N,] [rabbitmq_flush_interval_ms = N] + [rabbitmq_queue_settings_list = 'x-dead-letter-exchange=my-dlx,x-max-length=10,x-overflow=reject-publish'] ``` Обязательные параметры: @@ -51,6 +52,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `rabbitmq_skip_broken_messages` – максимальное количество некорректных сообщений в блоке. Если `rabbitmq_skip_broken_messages = N`, то движок отбрасывает `N` сообщений, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. - `rabbitmq_max_block_size` - `rabbitmq_flush_interval_ms` +- `rabbitmq_queue_settings_list` - позволяет самостоятельно установить любые настройки RabbitMQ при создании очереди Настройки форматов данных также могут быть добавлены в списке RabbitMQ настроек. From c17bf072fe4a5ddd5e180ebeac9b82d1705c1e30 Mon Sep 17 00:00:00 2001 From: olgarev Date: Mon, 1 Nov 2021 10:39:25 +0000 Subject: [PATCH 16/39] Initial --- .../en/engines/table-engines/special/merge.md | 50 +++++++++++-------- .../en/sql-reference/table-functions/merge.md | 17 ++++++- .../ru/engines/table-engines/special/merge.md | 40 +++++++++------ .../ru/sql-reference/table-functions/merge.md | 16 +++++- 4 files changed, 83 insertions(+), 40 deletions(-) diff --git a/docs/en/engines/table-engines/special/merge.md b/docs/en/engines/table-engines/special/merge.md index 19ce19fcc64..7895b14f9da 100644 --- a/docs/en/engines/table-engines/special/merge.md +++ b/docs/en/engines/table-engines/special/merge.md @@ -9,45 +9,54 @@ The `Merge` engine (not to be confused with `MergeTree`) does not store data its Reading is automatically parallelized. Writing to a table is not supported. When reading, the indexes of tables that are actually being read are used, if they exist. -The `Merge` engine accepts parameters: the database name and a regular expression for tables. - -## Examples {#examples} - -Example 1: +## Creating a Table {#creating-a-table} ``` sql -Merge(hits, '^WatchLog') + CREATE TABLE ... Engine=Merge(db_name, tables_regexp); ``` -Data will be read from the tables in the `hits` database that have names that match the regular expression ‘`^WatchLog`’. +**Engine Parameters** -Instead of the database name, you can use a constant expression that returns a string. For example, `currentDatabase()`. +- `db_name` — Database name or a regular expression to match the DB names. You can use a constant expression that returns a string, for example, `currentDatabase()`. + +- `tables_regexp` — A regular expression to match the table names in the specified DB or DBs. Regular expressions — [re2](https://github.com/google/re2) (supports a subset of PCRE), case-sensitive. -See the notes about escaping symbols in regular expressions in the “match” section. +See the notes about escaping symbols in regular expressions in the "match" section. -When selecting tables to read, the `Merge` table itself will not be selected, even if it matches the regex. This is to avoid loops. -It is possible to create two `Merge` tables that will endlessly try to read each others’ data, but this is not a good idea. +## Usage {#usage} + +When selecting tables to read, the `Merge` table itself is not selected, even if it matches the regex. This is to avoid loops. +It is possible to create two `Merge` tables that will endlessly try to read each others' data, but this is not a good idea. The typical way to use the `Merge` engine is for working with a large number of `TinyLog` tables as if with a single table. -Example 2: +## Examples {#examples} -Let’s say you have a old table (WatchLog_old) and decided to change partitioning without moving data to a new table (WatchLog_new) and you need to see data from both tables. +**Example 1** + +Consider two databases `ABC_corporate_site` and `ABC_store`. The `all_visitors` table will contain IDs from the tables `visitors` in both databases. ``` sql -CREATE TABLE WatchLog_old(date Date, UserId Int64, EventType String, Cnt UInt64) -ENGINE=MergeTree(date, (UserId, EventType), 8192); +CREATE TABLE all_visitors (id UInt32) ENGINE=Merge(REGEXP('ABC_*'), 'visitors'); +``` + +**Example 2** + +Let's say you have a old table `WatchLog_old` and decided to change partitioning without moving data to a new table `WatchLog_new`, and you need to see data from both tables. + +``` sql +CREATE TABLE WatchLog_old(date Date, UserId Int64, EventType String, Cnt UInt64) + ENGINE=MergeTree(date, (UserId, EventType), 8192); INSERT INTO WatchLog_old VALUES ('2018-01-01', 1, 'hit', 3); -CREATE TABLE WatchLog_new(date Date, UserId Int64, EventType String, Cnt UInt64) -ENGINE=MergeTree PARTITION BY date ORDER BY (UserId, EventType) SETTINGS index_granularity=8192; +CREATE TABLE WatchLog_new(date Date, UserId Int64, EventType String, Cnt UInt64) + ENGINE=MergeTree PARTITION BY date ORDER BY (UserId, EventType) SETTINGS index_granularity=8192; INSERT INTO WatchLog_new VALUES ('2018-01-02', 2, 'hit', 3); CREATE TABLE WatchLog as WatchLog_old ENGINE=Merge(currentDatabase(), '^WatchLog'); -SELECT * -FROM WatchLog +SELECT * FROM WatchLog; ``` ``` text @@ -68,5 +77,4 @@ FROM WatchLog **See Also** - [Virtual columns](../../../engines/table-engines/special/index.md#table_engines-virtual_columns) - -[Original article](https://clickhouse.com/docs/en/operations/table_engines/merge/) +- [merge](../../../sql-reference/table-functions/merge.md) table function diff --git a/docs/en/sql-reference/table-functions/merge.md b/docs/en/sql-reference/table-functions/merge.md index a5c74b71069..067df328e1e 100644 --- a/docs/en/sql-reference/table-functions/merge.md +++ b/docs/en/sql-reference/table-functions/merge.md @@ -5,7 +5,20 @@ toc_title: merge # merge {#merge} -`merge(db_name, 'tables_regexp')` – Creates a temporary Merge table. For more information, see the section “Table engines, Merge”. +Creates a temporary [Merge](../../engines/table-engines/special/merge.md) table. The table structure is taken from the first table encountered that matches the regular expression. -The table structure is taken from the first table encountered that matches the regular expression. +**Syntax** + +```sql +merge('db_name', 'tables_regexp') +``` +**Arguments** + +- `db_name` — Database name or a regular expression to match the DB names. You can use a constant expression that returns a string, for example, `currentDatabase()`. + +- `tables_regexp` — A regular expression to match the table names in the specified DB or DBs. + +**See Also** + +- [Merge](../../engines/table-engines/special/merge.md) table engine diff --git a/docs/ru/engines/table-engines/special/merge.md b/docs/ru/engines/table-engines/special/merge.md index 714b087c201..af82c347f7b 100644 --- a/docs/ru/engines/table-engines/special/merge.md +++ b/docs/ru/engines/table-engines/special/merge.md @@ -7,43 +7,53 @@ toc_title: Merge Движок `Merge` (не путайте с движком `MergeTree`) не хранит данные самостоятельно, а позволяет читать одновременно из произвольного количества других таблиц. Чтение автоматически распараллеливается. Запись в таблицу не поддерживается. При чтении будут использованы индексы тех таблиц, из которых реально идёт чтение, если они существуют. -Движок `Merge` принимает параметры: имя базы данных и регулярное выражение для таблиц. -Пример: +## Создание таблицы {#creating-a-table} ``` sql -Merge(hits, '^WatchLog') + CREATE TABLE ... Engine=Merge(db_name, tables_regexp); ``` -Данные будут читаться из таблиц в базе `hits`, имена которых соответствуют регулярному выражению ‘`^WatchLog`’. +**Параметры движка** -Вместо имени базы данных может использоваться константное выражение, возвращающее строку. Например, `currentDatabase()`. +- `db_name` — имя БД или регулярное выражение для отбора БД. Можно использовать выражение, возвращающее строку с именем БД, например, `currentDatabase()`. + +- `tables_regexp` — регулярное выражение для имен таблиц в указанной БД или нескольких БД. + +## Использование {#usage} Регулярные выражения — [re2](https://github.com/google/re2) (поддерживает подмножество PCRE), регистрозависимые. Смотрите замечание об экранировании в регулярных выражениях в разделе «match». -При выборе таблиц для чтения, сама `Merge`-таблица не будет выбрана, даже если попадает под регулярное выражение, чтобы не возникло циклов. -Впрочем, вы можете создать две `Merge`-таблицы, которые будут пытаться бесконечно читать данные друг друга, но делать этого не нужно. +При выборе таблиц для чтения сама `Merge`-таблица не будет выбрана, даже если попадает под регулярное выражение, чтобы не возникло циклов. +Впрочем, вы можете создать две `Merge`-таблицы, которые будут пытаться бесконечно читать данные друг друга, но делать этого не рекомендуется. -Типичный способ использования движка `Merge` — работа с большим количеством таблиц типа `TinyLog`, как с одной. +Типичный способ использования движка `Merge` — работа с большим количеством таблиц типа `TinyLog` как с одной. -Пример 2: +**Пример 1** + +Пусть есть две БД `ABC_corporate_site` и `ABC_store`. Таблица `all_visitors` будет содержать ID из таблиц `visitors` в обеих БД. + +``` sql +CREATE TABLE all_visitors (id UInt32) ENGINE=Merge(REGEXP('ABC_*'), 'visitors'); +``` + +**Пример 2** Пусть есть старая таблица `WatchLog_old`. Необходимо изменить партиционирование без перемещения данных в новую таблицу `WatchLog_new`. При этом в выборке должны участвовать данные обеих таблиц. ``` sql CREATE TABLE WatchLog_old(date Date, UserId Int64, EventType String, Cnt UInt64) -ENGINE=MergeTree(date, (UserId, EventType), 8192); + ENGINE=MergeTree(date, (UserId, EventType), 8192); INSERT INTO WatchLog_old VALUES ('2018-01-01', 1, 'hit', 3); CREATE TABLE WatchLog_new(date Date, UserId Int64, EventType String, Cnt UInt64) -ENGINE=MergeTree PARTITION BY date ORDER BY (UserId, EventType) SETTINGS index_granularity=8192; + ENGINE=MergeTree PARTITION BY date ORDER BY (UserId, EventType) SETTINGS index_granularity=8192; INSERT INTO WatchLog_new VALUES ('2018-01-02', 2, 'hit', 3); CREATE TABLE WatchLog as WatchLog_old ENGINE=Merge(currentDatabase(), '^WatchLog'); -SELECT * -FROM WatchLog +SELECT * FROM WatchLog; ``` ``` text @@ -61,7 +71,7 @@ FROM WatchLog В секции `WHERE/PREWHERE` можно установить константное условие на столбец `_table` (например, `WHERE _table='xyz'`). В этом случае операции чтения выполняются только для тех таблиц, для которых выполняется условие на значение `_table`, таким образом, столбец `_table` работает как индекс. -**Смотрите также** +**См. также** - [Виртуальные столбцы](index.md#table_engines-virtual_columns) - +- Табличная функция [merge](../../../sql-reference/table-functions/merge.md) diff --git a/docs/ru/sql-reference/table-functions/merge.md b/docs/ru/sql-reference/table-functions/merge.md index 5b33f458468..d57f3378e35 100644 --- a/docs/ru/sql-reference/table-functions/merge.md +++ b/docs/ru/sql-reference/table-functions/merge.md @@ -5,7 +5,19 @@ toc_title: merge # merge {#merge} -`merge(db_name, 'tables_regexp')` - создаёт временную таблицу типа Merge. Подробнее смотрите раздел «Движки таблиц, Merge». +Cоздаёт временную таблицу типа [Merge](../../engines/table-engines/special/merge.md). Структура таблицы берётся из первой попавшейся таблицы, подходящей под регулярное выражение. -Структура таблицы берётся из первой попавшейся таблицы, подходящей под регулярное выражение. +**Синтаксис** +```sql +merge('db_name', 'tables_regexp') +``` +**Аргументы** + +- `db_name` — имя БД или регулярное выражение для отбора БД. Можно использовать выражение, возвращающее строку с именем БД, например, `currentDatabase()`. + +- `tables_regexp` — ы + +**См. также** + +- Табличный движок [Merge](../../engines/table-engines/special/merge.md) From 7a2aa3ce54cff2c187f35e63d8f24a2fc81637f9 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 1 Nov 2021 19:26:02 +0300 Subject: [PATCH 17/39] Update rabbitmq.md --- docs/ru/engines/table-engines/integrations/rabbitmq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/integrations/rabbitmq.md b/docs/ru/engines/table-engines/integrations/rabbitmq.md index 77d69b4d0cc..12442834868 100644 --- a/docs/ru/engines/table-engines/integrations/rabbitmq.md +++ b/docs/ru/engines/table-engines/integrations/rabbitmq.md @@ -52,7 +52,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `rabbitmq_skip_broken_messages` – максимальное количество некорректных сообщений в блоке. Если `rabbitmq_skip_broken_messages = N`, то движок отбрасывает `N` сообщений, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. - `rabbitmq_max_block_size` - `rabbitmq_flush_interval_ms` -- `rabbitmq_queue_settings_list` - позволяет самостоятельно установить любые настройки RabbitMQ при создании очереди +- `rabbitmq_queue_settings_list` - позволяет самостоятельно настройки RabbitMQ при создании очереди. Доступные настройки: `x-max-length`, `x-max-length-bytes`, `x-message-ttl`, `x-expires`, `x-priority`, `x-max-priority`, `x-overflow`, `x-dead-letter-exchange`, `x-queue-type`. Настрока `durable` для очереди ставится автоматически. Настройки форматов данных также могут быть добавлены в списке RabbitMQ настроек. From da0f231a5f30be2fc24052502dc115844874b990 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Tue, 2 Nov 2021 12:01:01 +0300 Subject: [PATCH 18/39] Apply suggestions from code review Co-authored-by: gyuton <40863448+gyuton@users.noreply.github.com> --- docs/en/engines/table-engines/special/merge.md | 6 +++--- docs/ru/engines/table-engines/special/merge.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/table-engines/special/merge.md b/docs/en/engines/table-engines/special/merge.md index 7895b14f9da..3a72064b49f 100644 --- a/docs/en/engines/table-engines/special/merge.md +++ b/docs/en/engines/table-engines/special/merge.md @@ -12,7 +12,7 @@ Reading is automatically parallelized. Writing to a table is not supported. When ## Creating a Table {#creating-a-table} ``` sql - CREATE TABLE ... Engine=Merge(db_name, tables_regexp); + CREATE TABLE ... Engine=Merge(db_name, tables_regexp) ``` **Engine Parameters** @@ -35,7 +35,7 @@ The typical way to use the `Merge` engine is for working with a large number of **Example 1** -Consider two databases `ABC_corporate_site` and `ABC_store`. The `all_visitors` table will contain IDs from the tables `visitors` in both databases. +Consider two databases `ABC_corporate_site` and `ABC_store`. The `all_visitors` table will contain IDs from the tables `visitors` in both databases. ``` sql CREATE TABLE all_visitors (id UInt32) ENGINE=Merge(REGEXP('ABC_*'), 'visitors'); @@ -43,7 +43,7 @@ CREATE TABLE all_visitors (id UInt32) ENGINE=Merge(REGEXP('ABC_*'), 'visitors'); **Example 2** -Let's say you have a old table `WatchLog_old` and decided to change partitioning without moving data to a new table `WatchLog_new`, and you need to see data from both tables. +Let's say you have an old table `WatchLog_old` and decided to change partitioning without moving data to a new table `WatchLog_new`, and you need to see data from both tables. ``` sql CREATE TABLE WatchLog_old(date Date, UserId Int64, EventType String, Cnt UInt64) diff --git a/docs/ru/engines/table-engines/special/merge.md b/docs/ru/engines/table-engines/special/merge.md index af82c347f7b..b0288e530e1 100644 --- a/docs/ru/engines/table-engines/special/merge.md +++ b/docs/ru/engines/table-engines/special/merge.md @@ -11,7 +11,7 @@ toc_title: Merge ## Создание таблицы {#creating-a-table} ``` sql - CREATE TABLE ... Engine=Merge(db_name, tables_regexp); + CREATE TABLE ... Engine=Merge(db_name, tables_regexp) ``` **Параметры движка** From 59d6a7b790c1c9d3bcab8e1bb69862d71706e3bf Mon Sep 17 00:00:00 2001 From: olgarev Date: Tue, 2 Nov 2021 09:15:39 +0000 Subject: [PATCH 19/39] Fix --- docs/ru/sql-reference/table-functions/merge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/table-functions/merge.md b/docs/ru/sql-reference/table-functions/merge.md index d57f3378e35..1f83dee10e5 100644 --- a/docs/ru/sql-reference/table-functions/merge.md +++ b/docs/ru/sql-reference/table-functions/merge.md @@ -16,7 +16,7 @@ merge('db_name', 'tables_regexp') - `db_name` — имя БД или регулярное выражение для отбора БД. Можно использовать выражение, возвращающее строку с именем БД, например, `currentDatabase()`. -- `tables_regexp` — ы +- `tables_regexp` — регулярное выражение для имен таблиц в указанной БД или нескольких БД. **См. также** From 90f7f86ea8a6f345845c8c4ec8a84094d4dc2251 Mon Sep 17 00:00:00 2001 From: andrc1901 <92211164+andrc1901@users.noreply.github.com> Date: Tue, 2 Nov 2021 12:38:06 +0300 Subject: [PATCH 20/39] Update docs/ru/introduction/distinctive-features.md Co-authored-by: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> --- docs/ru/introduction/distinctive-features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/introduction/distinctive-features.md b/docs/ru/introduction/distinctive-features.md index b330b2a0b41..93e6a8d8e91 100644 --- a/docs/ru/introduction/distinctive-features.md +++ b/docs/ru/introduction/distinctive-features.md @@ -32,7 +32,7 @@ toc_title: "Отличительные возможности ClickHouse" ## Поддержка SQL {#sql-support} -ClickHouse поддерживает [декларативный язык запросов на основе SQL](../sql-reference/statements/select/group-by.md) и во [многих случаях](../sql-reference/ansi.md) совпадающий с SQL стандартом. +ClickHouse поддерживает [декларативный язык запросов на основе SQL](../sql-reference/index.md) и во [многих случаях](../sql-reference/ansi.md) совпадающий с SQL стандартом. Поддерживаются [GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), подзапросы в секциях [FROM](../sql-reference/statements/select/from.md), [IN](../sql-reference/operators/in.md), [JOIN](../sql-reference/statements/select/join.md), [функции window](../sql-reference/window-functions/index.md), а также скалярные подзапросы. From 394d982006ed886d0832f8e8976c77429e885cb1 Mon Sep 17 00:00:00 2001 From: Roman Chyrva Date: Tue, 2 Nov 2021 15:59:52 +0200 Subject: [PATCH 21/39] Add missed word --- docs/ru/engines/table-engines/integrations/rabbitmq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/integrations/rabbitmq.md b/docs/ru/engines/table-engines/integrations/rabbitmq.md index 12442834868..7322f23fe0e 100644 --- a/docs/ru/engines/table-engines/integrations/rabbitmq.md +++ b/docs/ru/engines/table-engines/integrations/rabbitmq.md @@ -52,7 +52,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `rabbitmq_skip_broken_messages` – максимальное количество некорректных сообщений в блоке. Если `rabbitmq_skip_broken_messages = N`, то движок отбрасывает `N` сообщений, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. - `rabbitmq_max_block_size` - `rabbitmq_flush_interval_ms` -- `rabbitmq_queue_settings_list` - позволяет самостоятельно настройки RabbitMQ при создании очереди. Доступные настройки: `x-max-length`, `x-max-length-bytes`, `x-message-ttl`, `x-expires`, `x-priority`, `x-max-priority`, `x-overflow`, `x-dead-letter-exchange`, `x-queue-type`. Настрока `durable` для очереди ставится автоматически. +- `rabbitmq_queue_settings_list` - позволяет самостоятельно установить настройки RabbitMQ при создании очереди. Доступные настройки: `x-max-length`, `x-max-length-bytes`, `x-message-ttl`, `x-expires`, `x-priority`, `x-max-priority`, `x-overflow`, `x-dead-letter-exchange`, `x-queue-type`. Настрока `durable` для очереди ставится автоматически. Настройки форматов данных также могут быть добавлены в списке RabbitMQ настроек. From dd2698ed98d5a8d6d19834b21c9ae2d1bbcdc7c8 Mon Sep 17 00:00:00 2001 From: olgarev Date: Tue, 2 Nov 2021 14:29:20 +0000 Subject: [PATCH 22/39] Possible values for db name --- docs/en/engines/table-engines/special/merge.md | 5 ++++- docs/en/sql-reference/table-functions/merge.md | 5 ++++- docs/ru/engines/table-engines/special/merge.md | 5 ++++- docs/ru/sql-reference/table-functions/merge.md | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/table-engines/special/merge.md b/docs/en/engines/table-engines/special/merge.md index 3a72064b49f..27f783a3cea 100644 --- a/docs/en/engines/table-engines/special/merge.md +++ b/docs/en/engines/table-engines/special/merge.md @@ -17,7 +17,10 @@ Reading is automatically parallelized. Writing to a table is not supported. When **Engine Parameters** -- `db_name` — Database name or a regular expression to match the DB names. You can use a constant expression that returns a string, for example, `currentDatabase()`. +- `db_name` — Possible values: + - database name, + - constant expression that returns a string with a database name, for example, `currentDatabase()`, + - `REGEXP(expression)`, where `expression` is a regular expression to match the DB names. - `tables_regexp` — A regular expression to match the table names in the specified DB or DBs. diff --git a/docs/en/sql-reference/table-functions/merge.md b/docs/en/sql-reference/table-functions/merge.md index 067df328e1e..c89f0f4cc5a 100644 --- a/docs/en/sql-reference/table-functions/merge.md +++ b/docs/en/sql-reference/table-functions/merge.md @@ -14,7 +14,10 @@ merge('db_name', 'tables_regexp') ``` **Arguments** -- `db_name` — Database name or a regular expression to match the DB names. You can use a constant expression that returns a string, for example, `currentDatabase()`. +- `db_name` — Possible values: + - database name, + - constant expression that returns a string with a database name, for example, `currentDatabase()`, + - `REGEXP(expression)`, where `expression` is a regular expression to match the DB names. - `tables_regexp` — A regular expression to match the table names in the specified DB or DBs. diff --git a/docs/ru/engines/table-engines/special/merge.md b/docs/ru/engines/table-engines/special/merge.md index b0288e530e1..47f8e871e47 100644 --- a/docs/ru/engines/table-engines/special/merge.md +++ b/docs/ru/engines/table-engines/special/merge.md @@ -16,7 +16,10 @@ toc_title: Merge **Параметры движка** -- `db_name` — имя БД или регулярное выражение для отбора БД. Можно использовать выражение, возвращающее строку с именем БД, например, `currentDatabase()`. +- `db_name` — Возможные варианты: + - имя БД, + - выражение, возвращающее строку с именем БД, например, `currentDatabase()`, + - `REGEXP(expression)`, где `expression` — регулярное выражение для отбора БД. - `tables_regexp` — регулярное выражение для имен таблиц в указанной БД или нескольких БД. diff --git a/docs/ru/sql-reference/table-functions/merge.md b/docs/ru/sql-reference/table-functions/merge.md index 1f83dee10e5..24246103d64 100644 --- a/docs/ru/sql-reference/table-functions/merge.md +++ b/docs/ru/sql-reference/table-functions/merge.md @@ -14,7 +14,10 @@ merge('db_name', 'tables_regexp') ``` **Аргументы** -- `db_name` — имя БД или регулярное выражение для отбора БД. Можно использовать выражение, возвращающее строку с именем БД, например, `currentDatabase()`. +- `db_name` — Возможные варианты: + - имя БД, + - выражение, возвращающее строку с именем БД, например, `currentDatabase()`, + - `REGEXP(expression)`, где `expression` — регулярное выражение для отбора БД. - `tables_regexp` — регулярное выражение для имен таблиц в указанной БД или нескольких БД. From bdd039a21c498653869e650598c08b92f7bb54aa Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 3 Nov 2021 19:31:50 +0300 Subject: [PATCH 23/39] Document the log_formatted_queries setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Задокументировал настройку. --- docs/en/operations/settings/settings.md | 10 +++ docs/en/operations/system-tables/query_log.md | 75 ++++++++++--------- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 92f397b2c39..40373f4ed0f 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -992,6 +992,16 @@ Example: log_query_views=1 ``` +## log_formatted_queries {#settings-log-formatted-queries} + +Allows a user to log formatted queries and to write to the [system.query_log](../../operations/system-tables/query_log.md) system table. + +Possible values: + +- 0 — Formatted queries are not logged. +- 1 — Formatted queries are logged. + +Default value: `0`. ## log_comment {#settings-log-comment} diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md index 0cadc975bd4..bd20fec7fa9 100644 --- a/docs/en/operations/system-tables/query_log.md +++ b/docs/en/operations/system-tables/query_log.md @@ -26,6 +26,8 @@ Each query creates one or two rows in the `query_log` table, depending on the st You can use the [log_queries_probability](../../operations/settings/settings.md#log-queries-probability) setting to reduce the number of queries, registered in the `query_log` table. +You can use the [log_formatted_queries](../../operations/settings/settings.md#settings-log-formatted-queries) setting to log formatted queries to the `formatted_query` column. + Columns: - `type` ([Enum8](../../sql-reference/data-types/enum.md)) — Type of an event that occurred when executing the query. Values: @@ -48,6 +50,7 @@ Columns: - `memory_usage` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Memory consumption by the query. - `current_database` ([String](../../sql-reference/data-types/string.md)) — Name of the current database. - `query` ([String](../../sql-reference/data-types/string.md)) — Query string. +- `formatted_query` ([String](../../sql-reference/data-types/string.md)) — Formatted_query string. - `normalized_query_hash` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Identical hash value without the values of literals for similar queries. - `query_kind` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) — Type of the query. - `databases` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the databases present in the query. @@ -114,68 +117,68 @@ SELECT * FROM system.query_log WHERE type = 'QueryFinish' ORDER BY query_start_t Row 1: ────── type: QueryFinish -event_date: 2021-07-28 -event_time: 2021-07-28 13:46:56 -event_time_microseconds: 2021-07-28 13:46:56.719791 -query_start_time: 2021-07-28 13:46:56 -query_start_time_microseconds: 2021-07-28 13:46:56.704542 -query_duration_ms: 14 -read_rows: 8393 -read_bytes: 374325 +event_date: 2021-11-03 +event_time: 2021-11-03 16:13:54 +event_time_microseconds: 2021-11-03 16:13:54.953024 +query_start_time: 2021-11-03 16:13:54 +query_start_time_microseconds: 2021-11-03 16:13:54.952325 +query_duration_ms: 0 +read_rows: 69 +read_bytes: 6187 written_rows: 0 written_bytes: 0 -result_rows: 4201 -result_bytes: 153024 -memory_usage: 4714038 +result_rows: 69 +result_bytes: 48256 +memory_usage: 0 current_database: default -query: SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res) -normalized_query_hash: 6666026786019643712 -query_kind: Select -databases: ['system'] -tables: ['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables'] -columns: ['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name'] +query: DESCRIBE TABLE system.query_log +formatted_query: +normalized_query_hash: 8274064835331539124 +query_kind: +databases: [] +tables: [] +columns: [] projections: [] +views: [] exception_code: 0 exception: stack_trace: is_initial_query: 1 user: default -query_id: a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf +query_id: 7c28bbbb-753b-4eba-98b1-efcbe2b9bdf6 address: ::ffff:127.0.0.1 -port: 51006 +port: 40452 initial_user: default -initial_query_id: a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf +initial_query_id: 7c28bbbb-753b-4eba-98b1-efcbe2b9bdf6 initial_address: ::ffff:127.0.0.1 -initial_port: 51006 -initial_query_start_time: 2021-07-28 13:46:56 -initial_query_start_time_microseconds: 2021-07-28 13:46:56.704542 +initial_port: 40452 +initial_query_start_time: 2021-11-03 16:13:54 +initial_query_start_time_microseconds: 2021-11-03 16:13:54.952325 interface: 1 -os_user: -client_hostname: -client_name: ClickHouse client +os_user: sevirov +client_hostname: clickhouse.ru-central1.internal +client_name: ClickHouse client_revision: 54449 client_version_major: 21 -client_version_minor: 8 -client_version_patch: 0 +client_version_minor: 10 +client_version_patch: 1 http_method: 0 http_user_agent: http_referer: forwarded_for: quota_key: -revision: 54453 +revision: 54456 log_comment: -thread_ids: [5058,22097,22110,22094] -ProfileEvents.Names: ['Query','SelectQuery','ArenaAllocChunks','ArenaAllocBytes','FunctionExecute','NetworkSendElapsedMicroseconds','SelectedRows','SelectedBytes','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SystemTimeMicroseconds','SoftPageFaults','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds','OSWriteBytes','OSWriteChars'] -ProfileEvents.Values: [1,1,39,352256,64,360,8393,374325,412,440,34480,13108,4723,671,19,17828,8192,10240] -Settings.Names: ['load_balancing','max_memory_usage'] -Settings.Values: ['random','10000000000'] +thread_ids: [30776,31174] +ProfileEvents: {'Query':1,'NetworkSendElapsedMicroseconds':59,'NetworkSendBytes':2643,'SelectedRows':69,'SelectedBytes':6187,'ContextLock':9,'RWLockAcquiredReadLocks':1,'RealTimeMicroseconds':817,'UserTimeMicroseconds':427,'SystemTimeMicroseconds':212,'OSCPUVirtualTimeMicroseconds':639,'OSReadChars':894,'OSWriteChars':319} +Settings: {'load_balancing':'random','max_memory_usage':'10000000000'} used_aggregate_functions: [] used_aggregate_function_combinators: [] used_database_engines: [] -used_data_type_families: ['UInt64','UInt8','Nullable','String','date'] +used_data_type_families: [] used_dictionaries: [] used_formats: [] -used_functions: ['concat','notEmpty','extractAll'] +used_functions: [] used_storages: [] used_table_functions: [] ``` From 234a5c37ac9ec931da5e2f40979f403a9fb42083 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 3 Nov 2021 21:03:46 +0300 Subject: [PATCH 24/39] Fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправил опечатку. --- docs/en/operations/system-tables/query_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md index bd20fec7fa9..b4e860c5fb0 100644 --- a/docs/en/operations/system-tables/query_log.md +++ b/docs/en/operations/system-tables/query_log.md @@ -50,7 +50,7 @@ Columns: - `memory_usage` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Memory consumption by the query. - `current_database` ([String](../../sql-reference/data-types/string.md)) — Name of the current database. - `query` ([String](../../sql-reference/data-types/string.md)) — Query string. -- `formatted_query` ([String](../../sql-reference/data-types/string.md)) — Formatted_query string. +- `formatted_query` ([String](../../sql-reference/data-types/string.md)) — Formatted query string. - `normalized_query_hash` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Identical hash value without the values of literals for similar queries. - `query_kind` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) — Type of the query. - `databases` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the databases present in the query. From 5481d3f144725b861842ad705a82196a9aa06daf Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 3 Nov 2021 21:28:36 +0300 Subject: [PATCH 25/39] Translate to Russian MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Выполнил перевод на русский язык. --- docs/en/operations/settings/settings.md | 4 +- docs/ru/operations/settings/settings.md | 11 +++ docs/ru/operations/system-tables/query_log.md | 75 ++++++++++--------- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 40373f4ed0f..4249f68febb 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -998,8 +998,8 @@ Allows a user to log formatted queries and to write to the [system.query_log](.. Possible values: -- 0 — Formatted queries are not logged. -- 1 — Formatted queries are logged. +- 0 — Formatted queries are not logged in the system table. +- 1 — Formatted queries are logged in the system table. Default value: `0`. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index e3f5f2e002d..71d53b2446a 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -922,6 +922,17 @@ log_queries_min_type='EXCEPTION_WHILE_PROCESSING' log_query_threads=1 ``` +## log_formatted_queries {#settings-log-formatted-queries} + +Позволяет пользователю регистрировать отформатированные запросы и записывать их в системную таблицу [system.query_log](../../operations/system-tables/query_log.md). + +Возможные значения: + +- 0 — отформатированные запросы не регистрируются в системной таблице. +- 1 — отформатированные запросы регистрируются в системной таблице. + +Значение по умолчанию: `0`. + ## log_comment {#settings-log-comment} Задаёт значение поля `log_comment` таблицы [system.query_log](../system-tables/query_log.md) и текст комментария в логе сервера. diff --git a/docs/ru/operations/system-tables/query_log.md b/docs/ru/operations/system-tables/query_log.md index f7709d6f3da..03050b75fb6 100644 --- a/docs/ru/operations/system-tables/query_log.md +++ b/docs/ru/operations/system-tables/query_log.md @@ -26,6 +26,8 @@ ClickHouse не удаляет данные из таблица автомати Чтобы уменьшить количество запросов, регистрирующихся в таблице `query_log`, вы можете использовать настройку [log_queries_probability](../../operations/settings/settings.md#log-queries-probability). +Чтобы регистрировать отформатированные запросы в столбце `formatted_query`, вы можете использовать настройку [log_formatted_queries](../../operations/settings/settings.md#settings-log-formatted-queries). + Столбцы: - `type` ([Enum8](../../sql-reference/data-types/enum.md)) — тип события, произошедшего при выполнении запроса. Значения: @@ -48,6 +50,7 @@ ClickHouse не удаляет данные из таблица автомати - `memory_usage` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — потребление RAM запросом. - `current_database` ([String](../../sql-reference/data-types/string.md)) — имя текущей базы данных. - `query` ([String](../../sql-reference/data-types/string.md)) — текст запроса. +- `formatted_query` ([String](../../sql-reference/data-types/string.md)) — текст отформатированного запроса. - `normalized_query_hash` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — идентичная хэш-сумма без значений литералов для аналогичных запросов. - `query_kind` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) — тип запроса. - `databases` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — имена баз данных, присутствующих в запросе. @@ -113,68 +116,68 @@ SELECT * FROM system.query_log WHERE type = 'QueryFinish' ORDER BY query_start_t Row 1: ────── type: QueryFinish -event_date: 2021-07-28 -event_time: 2021-07-28 13:46:56 -event_time_microseconds: 2021-07-28 13:46:56.719791 -query_start_time: 2021-07-28 13:46:56 -query_start_time_microseconds: 2021-07-28 13:46:56.704542 -query_duration_ms: 14 -read_rows: 8393 -read_bytes: 374325 +event_date: 2021-11-03 +event_time: 2021-11-03 16:13:54 +event_time_microseconds: 2021-11-03 16:13:54.953024 +query_start_time: 2021-11-03 16:13:54 +query_start_time_microseconds: 2021-11-03 16:13:54.952325 +query_duration_ms: 0 +read_rows: 69 +read_bytes: 6187 written_rows: 0 written_bytes: 0 -result_rows: 4201 -result_bytes: 153024 -memory_usage: 4714038 +result_rows: 69 +result_bytes: 48256 +memory_usage: 0 current_database: default -query: SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res) -normalized_query_hash: 6666026786019643712 -query_kind: Select -databases: ['system'] -tables: ['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables'] -columns: ['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name'] +query: DESCRIBE TABLE system.query_log +formatted_query: +normalized_query_hash: 8274064835331539124 +query_kind: +databases: [] +tables: [] +columns: [] projections: [] +views: [] exception_code: 0 exception: stack_trace: is_initial_query: 1 user: default -query_id: a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf +query_id: 7c28bbbb-753b-4eba-98b1-efcbe2b9bdf6 address: ::ffff:127.0.0.1 -port: 51006 +port: 40452 initial_user: default -initial_query_id: a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf +initial_query_id: 7c28bbbb-753b-4eba-98b1-efcbe2b9bdf6 initial_address: ::ffff:127.0.0.1 -initial_port: 51006 -initial_query_start_time: 2021-07-28 13:46:56 -initial_query_start_time_microseconds: 2021-07-28 13:46:56.704542 +initial_port: 40452 +initial_query_start_time: 2021-11-03 16:13:54 +initial_query_start_time_microseconds: 2021-11-03 16:13:54.952325 interface: 1 -os_user: -client_hostname: -client_name: ClickHouse client +os_user: sevirov +client_hostname: clickhouse.ru-central1.internal +client_name: ClickHouse client_revision: 54449 client_version_major: 21 -client_version_minor: 8 -client_version_patch: 0 +client_version_minor: 10 +client_version_patch: 1 http_method: 0 http_user_agent: http_referer: forwarded_for: quota_key: -revision: 54453 +revision: 54456 log_comment: -thread_ids: [5058,22097,22110,22094] -ProfileEvents.Names: ['Query','SelectQuery','ArenaAllocChunks','ArenaAllocBytes','FunctionExecute','NetworkSendElapsedMicroseconds','SelectedRows','SelectedBytes','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SystemTimeMicroseconds','SoftPageFaults','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds','OSWriteBytes','OSWriteChars'] -ProfileEvents.Values: [1,1,39,352256,64,360,8393,374325,412,440,34480,13108,4723,671,19,17828,8192,10240] -Settings.Names: ['load_balancing','max_memory_usage'] -Settings.Values: ['random','10000000000'] +thread_ids: [30776,31174] +ProfileEvents: {'Query':1,'NetworkSendElapsedMicroseconds':59,'NetworkSendBytes':2643,'SelectedRows':69,'SelectedBytes':6187,'ContextLock':9,'RWLockAcquiredReadLocks':1,'RealTimeMicroseconds':817,'UserTimeMicroseconds':427,'SystemTimeMicroseconds':212,'OSCPUVirtualTimeMicroseconds':639,'OSReadChars':894,'OSWriteChars':319} +Settings: {'load_balancing':'random','max_memory_usage':'10000000000'} used_aggregate_functions: [] used_aggregate_function_combinators: [] used_database_engines: [] -used_data_type_families: ['UInt64','UInt8','Nullable','String','date'] +used_data_type_families: [] used_dictionaries: [] used_formats: [] -used_functions: ['concat','notEmpty','extractAll'] +used_functions: [] used_storages: [] used_table_functions: [] ``` From 26c4affc185ae4ea764b9146fba887effc20a8fd Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 3 Nov 2021 09:17:27 +0000 Subject: [PATCH 26/39] Fixes --- programs/client/Client.cpp | 6 +++++- src/Client/ClientBase.cpp | 3 +++ ...02112_delayed_clickhouse_client_with_queries_file.expect | 3 --- .../02112_delayed_clickhouse_local.expect.reference | 0 .../02112_delayed_clickhouse_local_with_queries_file.expect | 3 --- 5 files changed, 8 insertions(+), 7 deletions(-) delete mode 100644 tests/queries/0_stateless/02112_delayed_clickhouse_local.expect.reference diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index c5db97a8958..0ad0764d721 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -408,6 +408,9 @@ void Client::prepareForInteractive() clearTerminal(); showClientVersion(); + if (delayed_interactive) + std::cout << std::endl; + /// Load Warnings at the beginning of connection if (!config().has("no-warnings")) { @@ -563,8 +566,9 @@ void Client::connect() if (is_interactive) { std::cout << "Connected to " << server_name << " server version " << server_version << " revision " << server_revision << "." - << std::endl << std::endl; + if (!delayed_interactive) + std::cout << std::endl; auto client_version_tuple = std::make_tuple(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH); auto server_version_tuple = std::make_tuple(server_version_major, server_version_minor, server_version_patch); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 76a704909c9..3866f8264f0 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1705,6 +1705,9 @@ void ClientBase::init(int argc, char ** argv) config().setString("history_file", options["history_file"].as()); if (options.count("verbose")) config().setBool("verbose", true); + if (options.count("interactive")) + config().setBool("interactive", true); + if (options.count("log-level")) Poco::Logger::root().setLevel(options["log-level"].as()); if (options.count("server_logs_file")) diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect index 245253ac205..b676c221c65 100755 --- a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect @@ -23,8 +23,5 @@ send -- "select * from t format TSV\r" expect "1" expect ":) " -send -- "exit\r" -expect eof - spawn bash -c "\$CLICKHOUSE_TESTS_DIR/helpers/02112_clean.sh" diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect.reference b/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect index 774d8d63714..f0aef1550c3 100755 --- a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect @@ -23,8 +23,5 @@ send -- "select * from t format TSV\r" expect "1" expect ":) " -send -- "exit\r" -expect eof - spawn bash -c "\$CLICKHOUSE_TESTS_DIR/helpers/02112_clean.sh" From f6e7fe67c4201193296aa328e1610712b7a99230 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Fri, 5 Nov 2021 10:44:41 +0000 Subject: [PATCH 27/39] improve tests --- ...048_parallel_reading_from_infile.reference | 8 ++++--- .../02048_parallel_reading_from_infile.sh | 22 +++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference index e356ab2095d..401ec7760e7 100644 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference @@ -1,4 +1,6 @@ +1 +2 +2 4 -4 -3 -3 +1 +2 diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh index 9c957305346..1484d7d2fa6 100755 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh @@ -4,23 +4,31 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -set -e - [ -e "${CLICKHOUSE_TMP}"/test_infile_parallel.gz ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel.gz [ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel [ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel_1 [ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel_2 + +echo -e "102\t2" > "${CLICKHOUSE_TMP}"/test_infile_parallel + +echo -e "102\tsecond" > "${CLICKHOUSE_TMP}"/test_infile_parallel_1 + +echo -e "103\tfirst" > "${CLICKHOUSE_TMP}"/test_infile_parallel_2 + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile_parallel;" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile_parallel (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory();" -${CLICKHOUSE_CLIENT} --query "SET input_format_tsv_enum_as_number = 1;" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel VALUES (102, 2), (103, 1);" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE '${CLICKHOUSE_TMP}/test_infile_parallel' FORMAT TSV;" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE '${CLICKHOUSE_TMP}/test_infile_parallel_1' FORMAT TSV;" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile_parallel INTO OUTFILE '${CLICKHOUSE_TMP}/test_infile_parallel_2' FORMAT TSV;" +${CLICKHOUSE_CLIENT} --query "SET input_format_tsv_enum_as_number = 0;" gzip "${CLICKHOUSE_TMP}"/test_infile_parallel +# Check that settings are applied +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT TSV;" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='first';" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='second';" + +${CLICKHOUSE_CLIENT} --query "SET input_format_tsv_enum_as_number = 1;" + ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT TSV;" ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='first';" ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='second';" From ae13237c55bc4d9438b7799dc274b746376e881e Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Fri, 5 Nov 2021 11:07:31 +0000 Subject: [PATCH 28/39] remove check for settings from test --- .../02048_parallel_reading_from_infile.reference | 2 -- .../0_stateless/02048_parallel_reading_from_infile.sh | 10 ---------- 2 files changed, 12 deletions(-) diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference index 401ec7760e7..099b7d91c92 100644 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference @@ -1,6 +1,4 @@ 1 2 -2 -4 1 2 diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh index 1484d7d2fa6..460c97cd5ac 100755 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh @@ -11,28 +11,18 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo -e "102\t2" > "${CLICKHOUSE_TMP}"/test_infile_parallel - echo -e "102\tsecond" > "${CLICKHOUSE_TMP}"/test_infile_parallel_1 - echo -e "103\tfirst" > "${CLICKHOUSE_TMP}"/test_infile_parallel_2 ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile_parallel;" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile_parallel (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory();" -${CLICKHOUSE_CLIENT} --query "SET input_format_tsv_enum_as_number = 0;" gzip "${CLICKHOUSE_TMP}"/test_infile_parallel -# Check that settings are applied ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT TSV;" ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='first';" ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='second';" -${CLICKHOUSE_CLIENT} --query "SET input_format_tsv_enum_as_number = 1;" - -${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT TSV;" -${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='first';" -${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='second';" - ${CLICKHOUSE_LOCAL} --multiquery < Date: Thu, 4 Nov 2021 08:58:31 -0600 Subject: [PATCH 29/39] Add overflow to benchmark tables # Conflicts: # website/css/main.css --- website/benchmark/dbms/index.html | 2 +- website/benchmark/hardware/index.html | 2 +- website/css/main.css | 2 +- website/src/scss/utilities/_overflow.scss | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 website/src/scss/utilities/_overflow.scss diff --git a/website/benchmark/dbms/index.html b/website/benchmark/dbms/index.html index 453b6fff9cd..b4e29098ead 100644 --- a/website/benchmark/dbms/index.html +++ b/website/benchmark/dbms/index.html @@ -35,7 +35,7 @@

Full results

-
+
diff --git a/website/benchmark/hardware/index.html b/website/benchmark/hardware/index.html index d7f8c80e1cf..6dc12890ef4 100644 --- a/website/benchmark/hardware/index.html +++ b/website/benchmark/hardware/index.html @@ -35,7 +35,7 @@

Full results

-
+
diff --git a/website/css/main.css b/website/css/main.css index d143d55e7ee..c7da0d486d7 100644 --- a/website/css/main.css +++ b/website/css/main.css @@ -1 +1 @@ -@media screen and (max-width:978.98px){.btn{padding:8px 16px}}@media screen and (max-width:978.98px){.btn-lg{padding:12px 24px}}.btn-primary,.btn-primary:active,.btn-primary:hover{color:#212529}.btn-outline-primary{background:#fffaf0;border-color:#fc0;color:#212529}.btn-outline-primary:active,.btn-outline-primary:hover{background:#fc0;border-color:#fc0;color:#212529}.btn-secondary{border-color:#212529;color:#fff}.btn-outline-secondary,.btn-secondary:active,.btn-secondary:hover{background:#fff;border-color:#212529;color:#212529}.btn-outline-secondary:active,.btn-outline-secondary:hover{background:#212529;border-color:#212529;color:#fff}.btn-tertiary{border-color:#257af4;color:#fff}.btn-tertiary:active,.btn-tertiary:hover{background:#257af4;border-color:#257af4;color:#fff}.btn-outline-tertiary{background:#e3f1fe;color:#257af4}.btn-outline-tertiary:active,.btn-outline-tertiary:hover{background:#257af4;color:#fff}.btns{align-items:center;display:grid;-moz-column-gap:24px;column-gap:24px;row-gap:16px;grid-auto-flow:column;justify-content:center}@media screen and (max-width:767.98px){.btns{grid-auto-flow:row}}.btns.btns-lg{-moz-column-gap:40px;column-gap:40px}.btns.is-2{grid-template-columns:1fr 1fr}@media screen and (max-width:767.98px){.btns.is-2{grid-template-columns:1fr}}.btns.is-3{grid-template-columns:1fr 1fr 1fr}@media screen and (max-width:767.98px){.btns.is-3{grid-template-columns:1fr}}.card{box-shadow:0 8px 20px rgba(108,117,125,.2);overflow:hidden;transition:box-shadow .2s,transform .2s;width:100%}.card,.card-body{position:relative}.card-body{z-index:10}.card.is-large .card-body{padding:40px}.card.bg-primary-light{border-color:#fc0}.card.has-dark-border{border-color:#6c757d}.card.has-pattern:after,.card.has-pattern:before{background-repeat:no-repeat;background-size:auto 100%;bottom:0;content:"";display:block;position:absolute;top:0;width:72px}.card.has-pattern:before{background-image:url(../images/backgrounds/bg-card-pattern-blue-1.png);background-position:0 0;left:0}.card.has-pattern:after{background-image:url(../images/backgrounds/bg-card-pattern-blue-2.png);background-position:100% 0;right:0}.card.has-hover:active,.card.has-hover:hover,a.card:active,a.card:hover{box-shadow:0 12px 32px rgba(108,117,125,.2);transform:translateY(-8px)}.card.has-highlight:after,.card.has-hover:after,a.card:after{content:"";display:block;height:8px;margin-top:auto;transition:background .2s;width:100%}.card.has-highlight:after,.card.has-hover:active:after,.card.has-hover:hover:after,a.card:active:after,a.card:hover:after{background:#e3f1fe}.case-study-cards{-moz-column-gap:40px;column-gap:40px;display:grid;grid-template-columns:1fr;row-gap:40px;padding-bottom:40px;position:relative}.case-study-cards>div{align-items:stretch;display:flex}.case-study-cards:before{background:#d6dbdf;bottom:0;content:"";display:block;left:20px;position:absolute;top:40px;width:100vw}@media screen and (min-width:980px){.case-study-cards{grid-template-columns:repeat(2,minmax(0,1fr));row-gap:80px;padding-bottom:120px}.case-study-cards:before{left:-40px;top:120px}}.case-study-card{align-items:stretch;flex-direction:row;flex-shrink:0;left:0;transition:box-shadow .2s,left .4s,width .4s,z-index 0s;transition-delay:0s,.6s,.6s,0s;width:100%;z-index:2}@media screen and (max-width:979.98px){.case-study-card .row{min-height:0!important}}@media screen and (min-width:980px){.case-study-card:active,.case-study-card:hover{box-shadow:0 12px 32px rgba(108,117,125,.2)}.case-study-card:not(.is-open){cursor:pointer}.case-study-card.is-open{transform:none!important;transition-delay:0s,0s,0s,0s;width:calc(200% + 40px);z-index:10}.case-study-card.is-closing{z-index:10}.case-study-card.open-left.is-open{left:calc(-100% - 40px)}.case-study-card:before{background:no-repeat url(../images/backgrounds/bg-card-pattern-red.png);background-position:100%;background-size:contain;content:"";display:block;height:calc(100% - 80px);max-height:224px;max-width:234px;position:absolute;right:0;top:40px;transform:translateX(30%);transition:transform .4s;transition-delay:.6s;width:100%;z-index:1}}@media screen and (min-width:980px)and (min-width:1240px){.case-study-card:before{transform:translateX(10%)}}@media screen and (min-width:980px){.case-study-card.is-open:before{transform:translateX(60%);transition-delay:0s}}@media screen and (min-width:980px){.case-study-card-wrap{align-items:stretch;display:flex;flex-shrink:0;min-height:304px;position:relative;transition:width .4s;transition-delay:.6s;width:calc(200% + 42px);z-index:2}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-wrap{transition-delay:0s;width:100%}}@media screen and (min-width:980px){.case-study-card-body{display:flex;flex-direction:column;padding-right:80px!important}.case-study-card-body>.row{align-self:stretch}}@media screen and (min-width:980px){.case-study-card-toggle{background:#fff;box-shadow:0 8px 20px rgba(108,117,125,.2);border-radius:100%;cursor:pointer;height:56px;position:relative;width:56px}.case-study-card-toggle:after,.case-study-card-toggle:before{background:#257af4;content:"";display:block;height:4px;left:calc(50% - 15px);position:absolute;top:calc(50% - 2px);transition:opacity .2s,transform .2s;width:30px}.case-study-card-toggle:after{transform:rotate(90deg)}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-toggle:before{opacity:0;transform:rotate(-90deg)}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-toggle:after{transform:rotate(0)}}@media screen and (min-width:980px){.case-study-card .col-lg-3,.case-study-card .col-lg-auto{opacity:0;transform:translateX(24px);transition:opacity .4s,transform .4s}}@media screen and (min-width:980px){.case-study-card .col-lg-3{transition-delay:0s}}@media screen and (min-width:980px){.case-study-card .col-lg-auto{transition-delay:.2s}}@media screen and (min-width:980px)and (min-width:980px){.case-study-card .col-lg-auto{max-width:605px;width:calc(100% - 319px)}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-3,.case-study-card.is-open .col-lg-auto{opacity:1;transform:none}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-3{transition-delay:.4s}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-auto{transition-delay:.2s}}.footer-copy{white-space:nowrap}form .form-control{border:1px solid #6c757d;border-radius:6px;height:auto;line-height:20px;min-height:44px;padding:12px 16px;width:100%}form .form-control,form .form-control:focus{box-shadow:0 8px 20px rgba(108,117,125,.2);color:#212529}form .form-control:focus{border-color:#212529}form .form-control::-moz-placeholder{color:#6c757d}form .form-control:-ms-input-placeholder{color:#6c757d}form .form-control::placeholder{color:#6c757d}form select.form-control{-webkit-appearance:none;-moz-appearance:none;appearance:none}form select.form-control:not([data-chosen]){color:#6c757d}form .btn-secondary:active,form .btn-secondary:hover{color:#212529;background:#fc0;border-color:#fc0}.hero{overflow:visible;position:relative}.hero,.hero-bg{background-repeat:no-repeat;background-position:50%;background-size:cover}.hero-bg{display:block;height:100%;left:50%;position:absolute;top:0;transform:translateX(-50%);z-index:1}.hero>.container{position:relative;z-index:2}.hero.has-offset{margin-bottom:-160px;padding-bottom:160px}.base-hero{height:22.5vw;max-height:324px;min-height:280px}.index-hero{background-image:url(/images/backgrounds/bg-hero-home.svg);height:68vw;max-height:980px}.index-hero,.other-hero{max-width:2448px;width:170vw}.other-hero{background-image:url(/images/backgrounds/bg-hero.svg)}.bg-footer-cta{background-image:url(/images/backgrounds/bg-footer-cta.svg);width:2448px}.quickstart-bg{background-image:url(/images/backgrounds/bg-quick-start.svg);height:40vw;top:220px;width:170vw}hr{background:#f1f6f9;border:0;display:block;height:4px;margin:0;width:100%}hr.is-small{height:2px}hr.is-large{height:8px}hr.is-medium{background:#d6dbdf}hr.is-dark{background:#495057}hr.is-yellow{background:linear-gradient(90deg,#ff8c00,#ff8c00 8px,#fc0 16px,rgba(255,204,0,0));-webkit-clip-path:polygon(8px 100%,0 100%,0 0,8px 0,8px 100%,16px 100%,16px 0,100% 0,100% 100%);clip-path:polygon(8px 100%,0 100%,0 0,8px 0,8px 100%,16px 100%,16px 0,100% 0,100% 100%);height:8px}.icon{display:block;height:48px;margin-bottom:24px;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center}@media screen and (min-width:576px){.icon{height:64px}}@media screen and (min-width:980px){.icon{height:80px}}img{max-width:100%}.kicker{color:#6c757d;font-family:Hind Siliguri,sans-serif;font-size:.875rem;font-weight:600;letter-spacing:1px;margin:0}@media screen and (max-width:978.98px){.lead{font-size:1.125rem}}.logo{display:block;height:36px;max-width:220px;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center;width:100%}.navbar-clickhouse{border-bottom:4px solid #f1f6f9;height:142px}.navbar-clickhouse>.container{flex-wrap:wrap}.navbar-super{flex-shrink:0;width:100%}.navbar-super ul{list-style:none}.navbar-super li:not(:last-child){margin-bottom:0;margin-right:24px}.navbar-super a{align-items:center;color:#212529;display:flex;font-size:.875rem}.navbar-super a:active,.navbar-super a:hover{color:#257af4;text-decoration:none}.navbar-super img{flex-shrink:0;margin-right:4px}.navbar-brand-clickhouse{background:no-repeat url(../images/logo-clickhouse.svg);background-size:contain;flex-shrink:0;height:28px;margin-right:48px;padding:0;width:180px}.navbar-nav{align-items:center;height:46px}.navbar .nav-item:not(:last-child){margin-bottom:0;margin-right:24px}.navbar .nav-link{color:#212529}.navbar .nav-link:active,.navbar .nav-link:hover{color:#257af4}.navbar .navbar-nav{flex-direction:row}@media screen and (max-width:978.98px){.navbar>.container{padding-left:20px;padding-right:20px}.navbar .navbar-toggler{height:24px;padding:0;width:24px}.navbar .navbar-toggler:focus{outline:none}.navbar .navbar-toggler-icon{background:no-repeat url(../images/icons/icon-menu.svg);background-position:50%;background-size:contain;height:24px;width:24px}.navbar .navbar-collapse{background:#fff;border-bottom:4px solid #f1f6f9;height:56px;left:0;padding:0 20px 16px;position:absolute;right:0;top:100%}.navbar .nav-link{font-size:.875rem;white-space:nowrap}}@media screen and (max-width:615.98px){.navbar .navbar-collapse{height:auto}.navbar .navbar-nav{flex-direction:column;height:auto}.navbar .nav-item:not(:last-child){margin-bottom:16px;margin-right:0}}@media screen and (max-width:399.98px){.navbar{height:80px}}.page,.photo-frame{overflow:hidden;width:100%}.photo-frame{background:hsla(0,0%,100%,.6);border-radius:100%;box-shadow:0 8px 20px rgba(108,117,125,.2);display:block;margin-bottom:24px;max-width:160px;position:relative}.photo-frame:before{content:"";display:block;padding-bottom:100%;width:100%}.photo-frame img{display:block;height:100%;left:0;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center;position:absolute;top:0;width:100%}.pullquote{position:relative;width:70%}.pullquote:before{background:no-repeat url(../images/backgrounds/bg-quotes.svg);background-position:50%;background-size:100%;content:"";mix-blend-mode:multiply;right:56px;width:calc(100% - 16px);z-index:2}.pullquote-bg,.pullquote:before{bottom:0;display:block;position:absolute;top:0}.pullquote-bg{right:0;width:calc(50vw + 28.57143%);z-index:1}.pullquote-body{padding:64px 40px 64px 0;position:relative;z-index:3}.pullquote-quote{font-family:Hind Siliguri,sans-serif;font-size:32px;font-weight:700}.pullquote-citation{font-size:1.125rem}.section{overflow:visible;position:relative}.section,.section-bg{background-repeat:no-repeat;background-position:50%;background-size:cover}.section-bg{display:block;height:100%;left:50%;position:absolute;top:0;transform:translateX(-50%);z-index:1}.section>.container{position:relative;z-index:2}.social-icons{align-items:center;display:flex}.social-icons>a{aspect-ratio:24/24;background:#6c757d;display:block;height:24px;width:24px;-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;transition:background .2s}.social-icons>a:active,.social-icons>a:hover{background:#212529}.social-icons>a+a{margin-left:32px}.social-icons-facebook{-webkit-mask-image:url(/images/icons/icon-facebook-gray.svg);mask-image:url(/images/icons/icon-facebook-gray.svg)}.social-icons-twitter{-webkit-mask-image:url(/images/icons/icon-twitter-gray.svg);mask-image:url(/images/icons/icon-twitter-gray.svg);width:31px}.social-icons-linkedin{-webkit-mask-image:url(/images/icons/icon-linkedin-gray.svg);mask-image:url(/images/icons/icon-linkedin-gray.svg)}.social-icons-linkedin-alt{-webkit-mask-image:url(/images/icons/icon-linkedin-alt-gray.svg);mask-image:url(/images/icons/icon-linkedin-alt-gray.svg)}.social-icons.size-small>a{height:20px;width:20px}.social-icons.size-small>a:active,.social-icons.size-small>a:hover{background:#212529}.social-icons.size-small>a+a{margin-left:16px}.tabs{position:relative}.tabs:before{background:#fff;border-radius:7px 7px 0 0;content:"";display:block;height:8px;left:1px;position:absolute;right:1px;top:68px;z-index:10}@media screen and (min-width:1240px){.tabs:before{top:76px}}.tabs-body{background:#fff;border-radius:8px;border:1px solid #6c757d;box-shadow:0 8px 20px rgba(108,117,125,.2);padding:24px}@media screen and (min-width:980px){.tabs-body{padding:32px}}@media screen and (min-width:1240px){.tabs-body{padding:40px}}.tabs .nav-tabs{border-bottom:0;flex-wrap:nowrap;height:76px;margin:-20px -20px -9px;-webkit-mask-image:linear-gradient(90deg,transparent,#000 20px,#000 calc(100% - 20px),transparent);mask-image:linear-gradient(90deg,transparent,#000 20px,#000 calc(100% - 20px),transparent);overflow:scroll;overflow-x:scroll;overflow-y:visible;padding:20px 20px 0;position:relative}@media screen and (min-width:940px){.tabs .nav-tabs{overflow:visible}}@media screen and (min-width:1240px){.tabs .nav-tabs{height:84px}}.tabs .nav-link{align-items:center;border-bottom:0;color:#6c757d;display:flex;font-size:.875rem;flex-shrink:0;height:56px;justify-content:center;padding:0 12px 8px;text-align:center;white-space:nowrap}@media screen and (min-width:1240px){.tabs .nav-link{height:64px;padding:0 16px 8px}}.tabs .nav-link.active{background:#fff;box-shadow:0 -4px 8px rgba(108,117,125,.1);font-weight:700;padding:0 16px 8px}@media screen and (min-width:980px){.tabs .nav-link.active{padding:0 24px 8px}}@media screen and (min-width:1240px){.tabs .nav-link.active{padding:0 32px 8px}}.tab-pane pre{background:#212529;border-radius:16px;color:#fff;padding:24px 16px}@media screen and (min-width:1240px){.tab-pane pre{padding:32px 24px}}.trailing-link{align-items:center;color:#212529;display:flex;font-size:.875rem;font-weight:700}.trailing-link:after{background:no-repeat url(../images/icons/icon-arrow.svg);background-position:100%;background-size:contain;content:"";display:block;height:12px;transition:transform .2s;width:20px}.trailing-link:active,.trailing-link:hover{color:#212529;text-decoration:none}.trailing-link:active:after,.trailing-link:hover:after{transform:translateX(8px)}.trailing-link.span-full:after{margin-left:auto}ul{color:#495057;list-style-type:square;padding-left:1.25em}ul li:not(:last-child){margin-bottom:16px}ul li::marker{color:#ff3939}ul.has-separators{list-style:none;padding:0}ul.has-separators li:not(:last-child){border-bottom:4px solid #f1f6f9;margin-bottom:24px;padding-bottom:24px}.bg-gradient-secondary{background-image:linear-gradient(58deg,#ff6443 3%,#fe561d 24%,#e32f0d 93%)}.bg-gradient-light-orange{background-image:linear-gradient(90deg,rgba(255,203,128,0),#ffcb80)}.bg-offset-right{bottom:0;left:-24px;position:absolute;top:0;width:calc(100vw + 24px);z-index:-1}@media screen and (min-width:1240px){.bg-offset-right{left:-96px;width:calc(100vw + 96px)}}.bg-inset-right{bottom:0;left:40px;position:absolute;top:0;width:calc(100vw - 40px);z-index:-1}@media screen and (min-width:980px){.bg-inset-right{left:96px;width:calc(100vw - 96px)}}.has-border-left{border-left:8px solid #f1f6f9;padding-left:16px}.font-xl{font-size:1.25rem}.font-lg{font-size:1.125rem}.font-sm{font-size:.875rem}.font-xs{font-size:.625rem}.font-weight-semibold{font-weight:600}.display-5{color:#212529;font-size:20px;font-weight:500}.display-6{color:#212529;font-size:14px;font-weight:700}.text-decoration-underline{text-decoration:underline}.text-upper{text-transform:uppercase} \ No newline at end of file +@media screen and (max-width:978.98px){.btn{padding:8px 16px}}@media screen and (max-width:978.98px){.btn-lg{padding:12px 24px}}.btn-primary,.btn-primary:active,.btn-primary:hover{color:#212529}.btn-outline-primary{background:#fffaf0;border-color:#fc0;color:#212529}.btn-outline-primary:active,.btn-outline-primary:hover{background:#fc0;border-color:#fc0;color:#212529}.btn-secondary{border-color:#212529;color:#fff}.btn-outline-secondary,.btn-secondary:active,.btn-secondary:hover{background:#fff;border-color:#212529;color:#212529}.btn-outline-secondary:active,.btn-outline-secondary:hover{background:#212529;border-color:#212529;color:#fff}.btn-tertiary{border-color:#257af4;color:#fff}.btn-tertiary:active,.btn-tertiary:hover{background:#257af4;border-color:#257af4;color:#fff}.btn-outline-tertiary{background:#e3f1fe;color:#257af4}.btn-outline-tertiary:active,.btn-outline-tertiary:hover{background:#257af4;color:#fff}.btns{align-items:center;display:grid;-moz-column-gap:24px;column-gap:24px;row-gap:16px;grid-auto-flow:column;justify-content:center}@media screen and (max-width:767.98px){.btns{grid-auto-flow:row}}.btns.btns-lg{-moz-column-gap:40px;column-gap:40px}.btns.is-2{grid-template-columns:1fr 1fr}@media screen and (max-width:767.98px){.btns.is-2{grid-template-columns:1fr}}.btns.is-3{grid-template-columns:1fr 1fr 1fr}@media screen and (max-width:767.98px){.btns.is-3{grid-template-columns:1fr}}.card{box-shadow:0 8px 20px rgba(108,117,125,.2);overflow:hidden;transition:box-shadow .2s,transform .2s;width:100%}.card,.card-body{position:relative}.card-body{z-index:10}.card.is-large .card-body{padding:40px}.card.bg-primary-light{border-color:#fc0}.card.has-dark-border{border-color:#6c757d}.card.has-pattern:after,.card.has-pattern:before{background-repeat:no-repeat;background-size:auto 100%;bottom:0;content:"";display:block;position:absolute;top:0;width:72px}.card.has-pattern:before{background-image:url(../images/backgrounds/bg-card-pattern-blue-1.png);background-position:0 0;left:0}.card.has-pattern:after{background-image:url(../images/backgrounds/bg-card-pattern-blue-2.png);background-position:100% 0;right:0}.card.has-hover:active,.card.has-hover:hover,a.card:active,a.card:hover{box-shadow:0 12px 32px rgba(108,117,125,.2);transform:translateY(-8px)}.card.has-highlight:after,.card.has-hover:after,a.card:after{content:"";display:block;height:8px;margin-top:auto;transition:background .2s;width:100%}.card.has-highlight:after,.card.has-hover:active:after,.card.has-hover:hover:after,a.card:active:after,a.card:hover:after{background:#e3f1fe}.case-study-cards{-moz-column-gap:40px;column-gap:40px;display:grid;grid-template-columns:1fr;row-gap:40px;padding-bottom:40px;position:relative}.case-study-cards>div{align-items:stretch;display:flex}.case-study-cards:before{background:#d6dbdf;bottom:0;content:"";display:block;left:20px;position:absolute;top:40px;width:100vw}@media screen and (min-width:980px){.case-study-cards{grid-template-columns:repeat(2,minmax(0,1fr));row-gap:80px;padding-bottom:120px}.case-study-cards:before{left:-40px;top:120px}}.case-study-card{align-items:stretch;flex-direction:row;flex-shrink:0;left:0;transition:box-shadow .2s,left .4s,width .4s,z-index 0s;transition-delay:0s,.6s,.6s,0s;width:100%;z-index:2}@media screen and (max-width:979.98px){.case-study-card .row{min-height:0!important}}@media screen and (min-width:980px){.case-study-card:active,.case-study-card:hover{box-shadow:0 12px 32px rgba(108,117,125,.2)}.case-study-card:not(.is-open){cursor:pointer}.case-study-card.is-open{transform:none!important;transition-delay:0s,0s,0s,0s;width:calc(200% + 40px);z-index:10}.case-study-card.is-closing{z-index:10}.case-study-card.open-left.is-open{left:calc(-100% - 40px)}.case-study-card:before{background:no-repeat url(../images/backgrounds/bg-card-pattern-red.png);background-position:100%;background-size:contain;content:"";display:block;height:calc(100% - 80px);max-height:224px;max-width:234px;position:absolute;right:0;top:40px;transform:translateX(30%);transition:transform .4s;transition-delay:.6s;width:100%;z-index:1}}@media screen and (min-width:980px)and (min-width:1240px){.case-study-card:before{transform:translateX(10%)}}@media screen and (min-width:980px){.case-study-card.is-open:before{transform:translateX(60%);transition-delay:0s}}@media screen and (min-width:980px){.case-study-card-wrap{align-items:stretch;display:flex;flex-shrink:0;min-height:304px;position:relative;transition:width .4s;transition-delay:.6s;width:calc(200% + 42px);z-index:2}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-wrap{transition-delay:0s;width:100%}}@media screen and (min-width:980px){.case-study-card-body{display:flex;flex-direction:column;padding-right:80px!important}.case-study-card-body>.row{align-self:stretch}}@media screen and (min-width:980px){.case-study-card-toggle{background:#fff;box-shadow:0 8px 20px rgba(108,117,125,.2);border-radius:100%;cursor:pointer;height:56px;position:relative;width:56px}.case-study-card-toggle:after,.case-study-card-toggle:before{background:#257af4;content:"";display:block;height:4px;left:calc(50% - 15px);position:absolute;top:calc(50% - 2px);transition:opacity .2s,transform .2s;width:30px}.case-study-card-toggle:after{transform:rotate(90deg)}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-toggle:before{opacity:0;transform:rotate(-90deg)}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-toggle:after{transform:rotate(0)}}@media screen and (min-width:980px){.case-study-card .col-lg-3,.case-study-card .col-lg-auto{opacity:0;transform:translateX(24px);transition:opacity .4s,transform .4s}}@media screen and (min-width:980px){.case-study-card .col-lg-3{transition-delay:0s}}@media screen and (min-width:980px){.case-study-card .col-lg-auto{transition-delay:.2s}}@media screen and (min-width:980px)and (min-width:980px){.case-study-card .col-lg-auto{max-width:605px;width:calc(100% - 319px)}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-3,.case-study-card.is-open .col-lg-auto{opacity:1;transform:none}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-3{transition-delay:.4s}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-auto{transition-delay:.2s}}.footer-copy{white-space:nowrap}form .form-control{border:1px solid #6c757d;border-radius:6px;height:auto;line-height:20px;min-height:44px;padding:12px 16px;width:100%}form .form-control,form .form-control:focus{box-shadow:0 8px 20px rgba(108,117,125,.2);color:#212529}form .form-control:focus{border-color:#212529}form .form-control::-moz-placeholder{color:#6c757d}form .form-control:-ms-input-placeholder{color:#6c757d}form .form-control::placeholder{color:#6c757d}form select.form-control{-webkit-appearance:none;-moz-appearance:none;appearance:none}form select.form-control:not([data-chosen]){color:#6c757d}form .btn-secondary:active,form .btn-secondary:hover{color:#212529;background:#fc0;border-color:#fc0}.hero{overflow:visible;position:relative}.hero,.hero-bg{background-repeat:no-repeat;background-position:50%;background-size:cover}.hero-bg{display:block;height:100%;left:50%;position:absolute;top:0;transform:translateX(-50%);z-index:1}.hero>.container{position:relative;z-index:2}.hero.has-offset{margin-bottom:-160px;padding-bottom:160px}.base-hero{height:22.5vw;max-height:324px;min-height:280px}.index-hero{background-image:url(/images/backgrounds/bg-hero-home.svg);height:68vw;max-height:980px}.index-hero,.other-hero{max-width:2448px;width:170vw}.other-hero{background-image:url(/images/backgrounds/bg-hero.svg)}.bg-footer-cta{background-image:url(/images/backgrounds/bg-footer-cta.svg);width:2448px}.quickstart-bg{background-image:url(/images/backgrounds/bg-quick-start.svg);height:40vw;top:220px;width:170vw}hr{background:#f1f6f9;border:0;display:block;height:4px;margin:0;width:100%}hr.is-small{height:2px}hr.is-large{height:8px}hr.is-medium{background:#d6dbdf}hr.is-dark{background:#495057}hr.is-yellow{background:linear-gradient(90deg,#ff8c00,#ff8c00 8px,#fc0 16px,rgba(255,204,0,0));-webkit-clip-path:polygon(8px 100%,0 100%,0 0,8px 0,8px 100%,16px 100%,16px 0,100% 0,100% 100%);clip-path:polygon(8px 100%,0 100%,0 0,8px 0,8px 100%,16px 100%,16px 0,100% 0,100% 100%);height:8px}.icon{display:block;height:48px;margin-bottom:24px;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center}@media screen and (min-width:576px){.icon{height:64px}}@media screen and (min-width:980px){.icon{height:80px}}img{max-width:100%}.kicker{color:#6c757d;font-family:Hind Siliguri,sans-serif;font-size:.875rem;font-weight:600;letter-spacing:1px;margin:0}@media screen and (max-width:978.98px){.lead{font-size:1.125rem}}.logo{display:block;height:36px;max-width:220px;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center;width:100%}.navbar-clickhouse{border-bottom:4px solid #f1f6f9;height:142px}.navbar-clickhouse>.container{flex-wrap:wrap}.navbar-super{flex-shrink:0;width:100%}.navbar-super ul{list-style:none}.navbar-super li:not(:last-child){margin-bottom:0;margin-right:24px}.navbar-super a{align-items:center;color:#212529;display:flex;font-size:.875rem}.navbar-super a:active,.navbar-super a:hover{color:#257af4;text-decoration:none}.navbar-super img{flex-shrink:0;margin-right:4px}.navbar-brand-clickhouse{background:no-repeat url(../images/logo-clickhouse.svg);background-size:contain;flex-shrink:0;height:28px;margin-right:48px;padding:0;width:180px}.navbar-nav{align-items:center;height:46px}.navbar .nav-item:not(:last-child){margin-bottom:0;margin-right:24px}.navbar .nav-link{color:#212529}.navbar .nav-link:active,.navbar .nav-link:hover{color:#257af4}.navbar .navbar-nav{flex-direction:row}@media screen and (max-width:978.98px){.navbar>.container{padding-left:20px;padding-right:20px}.navbar .navbar-toggler{height:24px;padding:0;width:24px}.navbar .navbar-toggler:focus{outline:none}.navbar .navbar-toggler-icon{background:no-repeat url(../images/icons/icon-menu.svg);background-position:50%;background-size:contain;height:24px;width:24px}.navbar .navbar-collapse{background:#fff;border-bottom:4px solid #f1f6f9;height:56px;left:0;padding:0 20px 16px;position:absolute;right:0;top:100%}.navbar .nav-link{font-size:.875rem;white-space:nowrap}}@media screen and (max-width:615.98px){.navbar .navbar-collapse{height:auto}.navbar .navbar-nav{flex-direction:column;height:auto}.navbar .nav-item:not(:last-child){margin-bottom:16px;margin-right:0}}@media screen and (max-width:399.98px){.navbar{height:80px}}.page,.photo-frame{overflow:hidden;width:100%}.photo-frame{background:hsla(0,0%,100%,.6);border-radius:100%;box-shadow:0 8px 20px rgba(108,117,125,.2);display:block;margin-bottom:24px;max-width:160px;position:relative}.photo-frame:before{content:"";display:block;padding-bottom:100%;width:100%}.photo-frame img{display:block;height:100%;left:0;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center;position:absolute;top:0;width:100%}.pullquote{position:relative;width:70%}.pullquote:before{background:no-repeat url(../images/backgrounds/bg-quotes.svg);background-position:50%;background-size:100%;content:"";mix-blend-mode:multiply;right:56px;width:calc(100% - 16px);z-index:2}.pullquote-bg,.pullquote:before{bottom:0;display:block;position:absolute;top:0}.pullquote-bg{right:0;width:calc(50vw + 28.57143%);z-index:1}.pullquote-body{padding:64px 40px 64px 0;position:relative;z-index:3}.pullquote-quote{font-family:Hind Siliguri,sans-serif;font-size:32px;font-weight:700}.pullquote-citation{font-size:1.125rem}.section{overflow:visible;position:relative}.section,.section-bg{background-repeat:no-repeat;background-position:50%;background-size:cover}.section-bg{display:block;height:100%;left:50%;position:absolute;top:0;transform:translateX(-50%);z-index:1}.section>.container{position:relative;z-index:2}.social-icons{align-items:center;display:flex}.social-icons>a{aspect-ratio:24/24;background:#6c757d;display:block;height:24px;width:24px;-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;transition:background .2s}.social-icons>a:active,.social-icons>a:hover{background:#212529}.social-icons>a+a{margin-left:32px}.social-icons-facebook{-webkit-mask-image:url(/images/icons/icon-facebook-gray.svg);mask-image:url(/images/icons/icon-facebook-gray.svg)}.social-icons-twitter{-webkit-mask-image:url(/images/icons/icon-twitter-gray.svg);mask-image:url(/images/icons/icon-twitter-gray.svg);width:31px}.social-icons-linkedin{-webkit-mask-image:url(/images/icons/icon-linkedin-gray.svg);mask-image:url(/images/icons/icon-linkedin-gray.svg)}.social-icons-linkedin-alt{-webkit-mask-image:url(/images/icons/icon-linkedin-alt-gray.svg);mask-image:url(/images/icons/icon-linkedin-alt-gray.svg)}.social-icons.size-small>a{height:20px;width:20px}.social-icons.size-small>a:active,.social-icons.size-small>a:hover{background:#212529}.social-icons.size-small>a+a{margin-left:16px}.tabs{position:relative}.tabs:before{background:#fff;border-radius:7px 7px 0 0;content:"";display:block;height:8px;left:1px;position:absolute;right:1px;top:68px;z-index:10}@media screen and (min-width:1240px){.tabs:before{top:76px}}.tabs-body{background:#fff;border-radius:8px;border:1px solid #6c757d;box-shadow:0 8px 20px rgba(108,117,125,.2);padding:24px}@media screen and (min-width:980px){.tabs-body{padding:32px}}@media screen and (min-width:1240px){.tabs-body{padding:40px}}.tabs .nav-tabs{border-bottom:0;flex-wrap:nowrap;height:76px;margin:-20px -20px -9px;-webkit-mask-image:linear-gradient(90deg,transparent,#000 20px,#000 calc(100% - 20px),transparent);mask-image:linear-gradient(90deg,transparent,#000 20px,#000 calc(100% - 20px),transparent);overflow:scroll;overflow-x:scroll;overflow-y:visible;padding:20px 20px 0;position:relative}@media screen and (min-width:940px){.tabs .nav-tabs{overflow:visible}}@media screen and (min-width:1240px){.tabs .nav-tabs{height:84px}}.tabs .nav-link{align-items:center;border-bottom:0;color:#6c757d;display:flex;font-size:.875rem;flex-shrink:0;height:56px;justify-content:center;padding:0 12px 8px;text-align:center;white-space:nowrap}@media screen and (min-width:1240px){.tabs .nav-link{height:64px;padding:0 16px 8px}}.tabs .nav-link.active{background:#fff;box-shadow:0 -4px 8px rgba(108,117,125,.1);font-weight:700;padding:0 16px 8px}@media screen and (min-width:980px){.tabs .nav-link.active{padding:0 24px 8px}}@media screen and (min-width:1240px){.tabs .nav-link.active{padding:0 32px 8px}}.tab-pane pre{background:#212529;border-radius:16px;color:#fff;padding:24px 16px}@media screen and (min-width:1240px){.tab-pane pre{padding:32px 24px}}.trailing-link{align-items:center;color:#212529;display:flex;font-size:.875rem;font-weight:700}.trailing-link:after{background:no-repeat url(../images/icons/icon-arrow.svg);background-position:100%;background-size:contain;content:"";display:block;height:12px;transition:transform .2s;width:20px}.trailing-link:active,.trailing-link:hover{color:#212529;text-decoration:none}.trailing-link:active:after,.trailing-link:hover:after{transform:translateX(8px)}.trailing-link.span-full:after{margin-left:auto}ul{color:#495057;list-style-type:square;padding-left:1.25em}ul li:not(:last-child){margin-bottom:16px}ul li::marker{color:#ff3939}ul.has-separators{list-style:none;padding:0}ul.has-separators li:not(:last-child){border-bottom:4px solid #f1f6f9;margin-bottom:24px;padding-bottom:24px}.bg-gradient-secondary{background-image:linear-gradient(58deg,#ff6443 3%,#fe561d 24%,#e32f0d 93%)}.bg-gradient-light-orange{background-image:linear-gradient(90deg,rgba(255,203,128,0),#ffcb80)}.bg-offset-right{bottom:0;left:-24px;position:absolute;top:0;width:calc(100vw + 24px);z-index:-1}@media screen and (min-width:1240px){.bg-offset-right{left:-96px;width:calc(100vw + 96px)}}.bg-inset-right{bottom:0;left:40px;position:absolute;top:0;width:calc(100vw - 40px);z-index:-1}@media screen and (min-width:980px){.bg-inset-right{left:96px;width:calc(100vw - 96px)}}.has-border-left{border-left:8px solid #f1f6f9;padding-left:16px}.font-xl{font-size:1.25rem}.font-lg{font-size:1.125rem}.font-sm{font-size:.875rem}.font-xs{font-size:.625rem}.font-weight-semibold{font-weight:600}.display-5{color:#212529;font-size:20px;font-weight:500}.display-6{color:#212529;font-size:14px;font-weight:700}.overflow-auto{overflow:auto}.text-decoration-underline{text-decoration:underline}.text-upper{text-transform:uppercase} \ No newline at end of file diff --git a/website/src/scss/utilities/_overflow.scss b/website/src/scss/utilities/_overflow.scss new file mode 100644 index 00000000000..d4c8444371a --- /dev/null +++ b/website/src/scss/utilities/_overflow.scss @@ -0,0 +1,3 @@ +.overflow-auto { + overflow: auto; +} From 93b8863ceebf35c3c395156ba84f4ce599f392eb Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Fri, 5 Nov 2021 18:33:50 +0000 Subject: [PATCH 30/39] Add test for settings --- ...048_parallel_reading_from_infile.reference | 1 + .../02048_parallel_reading_from_infile.sh | 26 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference index 099b7d91c92..98c00d52990 100644 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.reference @@ -1,4 +1,5 @@ 1 2 +Correct 1 2 diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh index 460c97cd5ac..d53fe8dd305 100755 --- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh +++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh @@ -8,23 +8,35 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) [ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel [ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel_1 [ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel_2 - +[ -e "${CLICKHOUSE_TMP}"/test_infile_parallel ] && rm "${CLICKHOUSE_TMP}"/test_infile_parallel_3 echo -e "102\t2" > "${CLICKHOUSE_TMP}"/test_infile_parallel echo -e "102\tsecond" > "${CLICKHOUSE_TMP}"/test_infile_parallel_1 echo -e "103\tfirst" > "${CLICKHOUSE_TMP}"/test_infile_parallel_2 - -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile_parallel;" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile_parallel (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory();" +echo -e "103" > "${CLICKHOUSE_TMP}"/test_infile_parallel_3 gzip "${CLICKHOUSE_TMP}"/test_infile_parallel -${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT TSV;" -${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='first';" -${CLICKHOUSE_CLIENT} --query "SELECT count() FROM test_infile_parallel WHERE Value='second';" +${CLICKHOUSE_CLIENT} --multiquery <&1 | grep -q "36" && echo "Correct" || echo 'Fail' ${CLICKHOUSE_LOCAL} --multiquery < Date: Sat, 6 Nov 2021 11:07:41 +0300 Subject: [PATCH 31/39] Fix by comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправил согласно комментариям в PR. --- docs/en/operations/settings/settings.md | 2 +- docs/en/operations/system-tables/query_log.md | 3 --- docs/ru/operations/settings/settings.md | 2 +- docs/ru/operations/system-tables/query_log.md | 4 +--- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 4249f68febb..863111c3087 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -994,7 +994,7 @@ log_query_views=1 ## log_formatted_queries {#settings-log-formatted-queries} -Allows a user to log formatted queries and to write to the [system.query_log](../../operations/system-tables/query_log.md) system table. +Allows to log formatted queries to the [system.query_log](../../operations/system-tables/query_log.md) system table. Possible values: diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md index b4e860c5fb0..e3aab04f7dd 100644 --- a/docs/en/operations/system-tables/query_log.md +++ b/docs/en/operations/system-tables/query_log.md @@ -186,6 +186,3 @@ used_table_functions: [] **See Also** - [system.query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log) — This table contains information about each query execution thread. -- [system.query_views_log](../../operations/system-tables/query_views_log.md#system_tables-query_views_log) — This table contains information about each view executed during a query. - -[Original article](https://clickhouse.com/docs/en/operations/system-tables/query_log) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 71d53b2446a..255e743b11d 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -924,7 +924,7 @@ log_query_threads=1 ## log_formatted_queries {#settings-log-formatted-queries} -Позволяет пользователю регистрировать отформатированные запросы и записывать их в системную таблицу [system.query_log](../../operations/system-tables/query_log.md). +Позволяет регистрировать отформатированные запросы в системной таблице [system.query_log](../../operations/system-tables/query_log.md). Возможные значения: diff --git a/docs/ru/operations/system-tables/query_log.md b/docs/ru/operations/system-tables/query_log.md index 03050b75fb6..644cee853cc 100644 --- a/docs/ru/operations/system-tables/query_log.md +++ b/docs/ru/operations/system-tables/query_log.md @@ -182,8 +182,6 @@ used_storages: [] used_table_functions: [] ``` -**Смотрите также** +**См. также** - [system.query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log) — в этой таблице содержится информация о цепочке каждого выполненного запроса. - -[Оригинальная статья](https://clickhouse.com/docs/ru/operations/system_tables/query_log) From 54b8491a9562b91984530c45450a3a14402b530d Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 6 Nov 2021 09:54:19 +0000 Subject: [PATCH 32/39] single.md is removed before build in case it is left after previous abnormal build --- docs/tools/build.py | 3 +++ docs/tools/single_page.py | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index aa440ecb5dc..08329c33271 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -117,6 +117,9 @@ def build_for_lang(lang, args): ) ) + # Clean to be safe if last build finished abnormally + single_page.remove_temporary_files(lang, args) + raw_config['nav'] = nav.build_docs_nav(lang, args) cfg = config.load_config(**raw_config) diff --git a/docs/tools/single_page.py b/docs/tools/single_page.py index 0e82a1acb87..2c0ddebf3c7 100644 --- a/docs/tools/single_page.py +++ b/docs/tools/single_page.py @@ -12,6 +12,7 @@ import test import util import website +TEMPORARY_FILE_NAME = 'single.md' def recursive_values(item): if isinstance(item, dict): @@ -101,6 +102,14 @@ def concatenate(lang, docs_path, single_page_file, nav): single_page_file.flush() +def get_temporary_file_name(lang, args): + return os.path.join(args.docs_dir, lang, TEMPORARY_FILE_NAME) + +def remove_temporary_files(lang, args): + single_md_path = get_temporary_file_name(lang, args) + if os.path.exists(single_md_path): + os.unlink(single_md_path) + def build_single_page_version(lang, args, nav, cfg): logging.info(f'Building single page version for {lang}') @@ -109,7 +118,7 @@ def build_single_page_version(lang, args, nav, cfg): extra['single_page'] = True extra['is_amp'] = False - single_md_path = os.path.join(args.docs_dir, lang, 'single.md') + single_md_path = get_temporary_file_name(lang, args) with open(single_md_path, 'w') as single_md: concatenate(lang, args.docs_dir, single_md, nav) @@ -226,5 +235,4 @@ def build_single_page_version(lang, args, nav, cfg): logging.info(f'Finished building single page version for {lang}') - if os.path.exists(single_md_path): - os.unlink(single_md_path) + remove_temporary_files(lang, args) From 9b8f4bcff2238923b129a6a4ea5802e986d00836 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 7 Nov 2021 04:25:26 +0300 Subject: [PATCH 33/39] Update nyc-taxi.md --- docs/en/getting-started/example-datasets/nyc-taxi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/example-datasets/nyc-taxi.md b/docs/en/getting-started/example-datasets/nyc-taxi.md index b3233315db3..ff29fef7fe0 100644 --- a/docs/en/getting-started/example-datasets/nyc-taxi.md +++ b/docs/en/getting-started/example-datasets/nyc-taxi.md @@ -332,7 +332,7 @@ ORDER BY year, count(*) DESC The following server was used: -Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total,128 GiB RAM,8x6 TB HD on hardware RAID-5 +Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical cores total, 128 GiB RAM, 8x6 TB HD on hardware RAID-5 Execution time is the best of three runs. But starting from the second run, queries read data from the file system cache. No further caching occurs: the data is read out and processed in each run. From 6f39465736f40668b5eb1734afd9c0c54bcc8915 Mon Sep 17 00:00:00 2001 From: michon470 Date: Sun, 7 Nov 2021 08:59:52 +0300 Subject: [PATCH 34/39] Update clickhouse-keeper.md --- docs/en/operations/clickhouse-keeper.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/clickhouse-keeper.md b/docs/en/operations/clickhouse-keeper.md index 58c59ce9f79..81516140f84 100644 --- a/docs/en/operations/clickhouse-keeper.md +++ b/docs/en/operations/clickhouse-keeper.md @@ -21,7 +21,7 @@ By default, ClickHouse Keeper provides the same guarantees as ZooKeeper (lineari ClickHouse Keeper can be used as a standalone replacement for ZooKeeper or as an internal part of the ClickHouse server, but in both cases configuration is almost the same `.xml` file. The main ClickHouse Keeper configuration tag is ``. Keeper configuration has the following parameters: - `tcp_port` — Port for a client to connect (default for ZooKeeper is `2181`). -- `tcp_port_secure` — Secure port for a client to connect. +- `tcp_port_secure` — Secure port for an SSL connection between client and keeper-server. - `server_id` — Unique server id, each participant of the ClickHouse Keeper cluster must have a unique number (1, 2, 3, and so on). - `log_storage_path` — Path to coordination logs, better to store logs on the non-busy device (same for ZooKeeper). - `snapshot_storage_path` — Path to coordination snapshots. @@ -50,7 +50,11 @@ Internal coordination settings are located in `..` section and contain servers description. The only parameter for the whole quorum is `secure`, which enables encrypted connection for communication between quorum participants. The main parameters for each `` are: +Quorum configuration is located in `.` section and contain servers description. + +The only parameter for the whole quorum is `secure`, which enables encrypted connection for communication between quorum participants. The parameter can be set `true` if SSL connection is required for internal communication between nodes, or left unspecified otherwise. + +The main parameters for each `` are: - `id` — Server identifier in a quorum. - `hostname` — Hostname where this server is placed. From 8541c21aec90a323f53e9a4314d5c340db388677 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Sun, 7 Nov 2021 17:39:37 -0400 Subject: [PATCH 35/39] Doc. freeze with name --- docs/ru/sql-reference/statements/alter/partition.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/alter/partition.md b/docs/ru/sql-reference/statements/alter/partition.md index f875103a498..7cc6d5b31c1 100644 --- a/docs/ru/sql-reference/statements/alter/partition.md +++ b/docs/ru/sql-reference/statements/alter/partition.md @@ -165,7 +165,7 @@ ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr ## FREEZE PARTITION {#alter_freeze-partition} ``` sql -ALTER TABLE table_name FREEZE [PARTITION partition_expr] +ALTER TABLE table_name FREEZE [PARTITION partition_expr] [WITH NAME 'backup_name'] ``` Создаёт резервную копию для заданной партиции. Если выражение `PARTITION` опущено, резервные копии будут созданы для всех партиций. @@ -179,6 +179,7 @@ ALTER TABLE table_name FREEZE [PARTITION partition_expr] - `/var/lib/clickhouse/` — рабочая директория ClickHouse, заданная в конфигурационном файле; - `N` — инкрементальный номер резервной копии. +- если задан параметр `WITH NAME`, то вместо инкрементального номера используется значение параметра `'backup_name'`. !!! note "Примечание" При использовании [нескольких дисков для хранения данных таблицы](../../statements/alter/index.md#table_engine-mergetree-multiple-volumes) директория `shadow/N` появляется на каждом из дисков, на которых были куски, попавшие под выражение `PARTITION`. From b391ca512aea65e55bebe5b12313e7cbae5fd179 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Sun, 7 Nov 2021 17:42:56 -0400 Subject: [PATCH 36/39] Doc. freeze with name --- docs/en/sql-reference/statements/alter/partition.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index e1a76d2c0ae..12737624ecb 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -155,7 +155,7 @@ ALTER TABLE visits CLEAR COLUMN hour in PARTITION 201902 ## FREEZE PARTITION {#alter_freeze-partition} ``` sql -ALTER TABLE table_name FREEZE [PARTITION partition_expr] +ALTER TABLE table_name FREEZE [PARTITION partition_expr] [WITH NAME 'backup_name'] ``` This query creates a local backup of a specified partition. If the `PARTITION` clause is omitted, the query creates the backup of all partitions at once. @@ -169,6 +169,7 @@ At the time of execution, for a data snapshot, the query creates hardlinks to a - `/var/lib/clickhouse/` is the working ClickHouse directory specified in the config. - `N` is the incremental number of the backup. +- if the `WITH NAME` parameter is specified, then the value of the `'backup_name'` parameter is used instead of the incremental number. !!! note "Note" If you use [a set of disks for data storage in a table](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes), the `shadow/N` directory appears on every disk, storing data parts that matched by the `PARTITION` expression. From 4293ce7ca6673f9dafaab45e2c10478671d933a9 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 8 Nov 2021 03:07:07 +0300 Subject: [PATCH 37/39] Update tips.md --- docs/en/operations/tips.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md index 54c66bb8d13..19af7df8239 100644 --- a/docs/en/operations/tips.md +++ b/docs/en/operations/tips.md @@ -1,3 +1,4 @@ +--- toc_priority: 58 toc_title: Usage Recommendations --- From 299445ec7da10bd2ef62d8e333a95b7ab12bf5f2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 8 Nov 2021 03:08:36 +0300 Subject: [PATCH 38/39] Update tips.md --- docs/en/operations/tips.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md index 19af7df8239..cc27250b00d 100644 --- a/docs/en/operations/tips.md +++ b/docs/en/operations/tips.md @@ -72,8 +72,8 @@ For HDD, enable the write cache. ## File System {#file-system} Ext4 is the most reliable option. Set the mount options `noatime`. -XFS is also suitable, but it hasn’t been as thoroughly tested with ClickHouse. -Most other file systems should also work fine. File systems with delayed allocation work better. +XFS should be avoided. It works mostly fine but there are some reports about lower performance. +Most other file systems should also work fine. ## Linux Kernel {#linux-kernel} From ad81977acebfd39f34921373e23933373226e019 Mon Sep 17 00:00:00 2001 From: Amr Alaa Date: Mon, 8 Nov 2021 12:07:14 +0200 Subject: [PATCH 39/39] Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM (#30995) * Skip checkPartitionCanBeDropped check in case of ATTACH PARTITION ... FROM * Add integration test * Fix syntax error * Fix integration test * Fix integration test * Fix integration test * Skip max_partition_size_to_drop for MOVE PARTITION --- src/Storages/MergeTree/MergeTreeData.cpp | 4 +- .../__init__.py | 0 .../configs/config.xml | 4 ++ .../test.py | 50 +++++++++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_attach_partition_with_large_destination/__init__.py create mode 100644 tests/integration/test_attach_partition_with_large_destination/configs/config.xml create mode 100644 tests/integration/test_attach_partition_with_large_destination/test.py diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c5c14cd24a8..dd95e3eaee2 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3423,7 +3423,6 @@ Pipe MergeTreeData::alterPartition( case PartitionCommand::MoveDestinationType::TABLE: { - checkPartitionCanBeDropped(command.partition); String dest_database = query_context->resolveDatabase(command.to_database); auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table}, query_context); movePartitionToTable(dest_storage, command.partition, query_context); @@ -3445,7 +3444,8 @@ Pipe MergeTreeData::alterPartition( case PartitionCommand::REPLACE_PARTITION: { - checkPartitionCanBeDropped(command.partition); + if (command.replace) + checkPartitionCanBeDropped(command.partition); String from_database = query_context->resolveDatabase(command.from_database); auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table}, query_context); replacePartitionFrom(from_storage, command.partition, command.replace, query_context); diff --git a/tests/integration/test_attach_partition_with_large_destination/__init__.py b/tests/integration/test_attach_partition_with_large_destination/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_attach_partition_with_large_destination/configs/config.xml b/tests/integration/test_attach_partition_with_large_destination/configs/config.xml new file mode 100644 index 00000000000..0500e2ad554 --- /dev/null +++ b/tests/integration/test_attach_partition_with_large_destination/configs/config.xml @@ -0,0 +1,4 @@ + + 1 + 1 + diff --git a/tests/integration/test_attach_partition_with_large_destination/test.py b/tests/integration/test_attach_partition_with_large_destination/test.py new file mode 100644 index 00000000000..50f24f7a01e --- /dev/null +++ b/tests/integration/test_attach_partition_with_large_destination/test.py @@ -0,0 +1,50 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', main_configs=["configs/config.xml"], with_zookeeper=True) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def create_force_drop_flag(node): + force_drop_flag_path = "/var/lib/clickhouse/flags/force_drop_table" + node.exec_in_container(["bash", "-c", "touch {} && chmod a=rw {}".format(force_drop_flag_path, force_drop_flag_path)], user="root") + +@pytest.mark.parametrize("engine", ['Ordinary', 'Atomic']) +def test_attach_partition_with_large_destination(started_cluster, engine): + # Initialize + node.query("CREATE DATABASE db ENGINE={}".format(engine)) + node.query("CREATE TABLE db.destination (n UInt64) ENGINE=ReplicatedMergeTree('/test/destination', 'r1') ORDER BY n PARTITION BY n % 2") + node.query("CREATE TABLE db.source_1 (n UInt64) ENGINE=ReplicatedMergeTree('/test/source_1', 'r1') ORDER BY n PARTITION BY n % 2") + node.query("INSERT INTO db.source_1 VALUES (1), (2), (3), (4)") + node.query("CREATE TABLE db.source_2 (n UInt64) ENGINE=ReplicatedMergeTree('/test/source_2', 'r1') ORDER BY n PARTITION BY n % 2") + node.query("INSERT INTO db.source_2 VALUES (5), (6), (7), (8)") + + # Attach partition when destination partition is empty + node.query("ALTER TABLE db.destination ATTACH PARTITION 0 FROM db.source_1") + assert node.query("SELECT n FROM db.destination ORDER BY n") == "2\n4\n" + + # REPLACE PARTITION should still respect max_partition_size_to_drop + assert node.query_and_get_error("ALTER TABLE db.destination REPLACE PARTITION 0 FROM db.source_2") + assert node.query("SELECT n FROM db.destination ORDER BY n") == "2\n4\n" + + # Attach partition when destination partition is larger than max_partition_size_to_drop + node.query("ALTER TABLE db.destination ATTACH PARTITION 0 FROM db.source_2") + assert node.query("SELECT n FROM db.destination ORDER BY n") == "2\n4\n6\n8\n" + + # Cleanup + create_force_drop_flag(node) + node.query("DROP TABLE db.source_1 SYNC") + create_force_drop_flag(node) + node.query("DROP TABLE db.source_2 SYNC") + create_force_drop_flag(node) + node.query("DROP TABLE db.destination SYNC") + node.query("DROP DATABASE db") \ No newline at end of file